{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Action-Actor-Critic to Play Acrobot-v1\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:54:57 [INFO] env: <AcrobotEnv<Acrobot-v1>>\n",
      "21:54:57 [INFO] action_space: Discrete(3)\n",
      "21:54:57 [INFO] observation_space: Box(-28.274333953857422, 28.274333953857422, (6,), float32)\n",
      "21:54:57 [INFO] reward_range: (-inf, inf)\n",
      "21:54:57 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 15}\n",
      "21:54:57 [INFO] _max_episode_steps: 500\n",
      "21:54:57 [INFO] _elapsed_steps: None\n",
      "21:54:57 [INFO] id: Acrobot-v1\n",
      "21:54:57 [INFO] entry_point: gym.envs.classic_control:AcrobotEnv\n",
      "21:54:57 [INFO] reward_threshold: -100.0\n",
      "21:54:57 [INFO] nondeterministic: False\n",
      "21:54:57 [INFO] max_episode_steps: 500\n",
      "21:54:57 [INFO] _kwargs: {}\n",
      "21:54:57 [INFO] _env_name: Acrobot\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Acrobot-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class QActorCriticAgent:\n",
    "    def __init__(self, env):\n",
    "        self.action_n = env.action_space.n\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.actor_net = self.build_net(hidden_sizes=[100,],\n",
    "                output_size=self.action_n, output_activation=nn.softmax,\n",
    "                loss=losses.categorical_crossentropy,\n",
    "                learning_rate=0.0001)\n",
    "        self.critic_net = self.build_net(hidden_sizes=[100,],\n",
    "                output_size=self.action_n,\n",
    "                learning_rate=0.0002)\n",
    "\n",
    "    def build_net(self, hidden_sizes, output_size, input_size=None,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.01):\n",
    "        model = keras.Sequential()\n",
    "        for hidden_size in hidden_sizes:\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.discount = 1.\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        probs = self.actor_net.predict(observation[np.newaxis])[0]\n",
    "        action = np.random.choice(self.action_n, p=probs)\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                self.learn()\n",
    "            self.discount *= self.gamma\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def learn(self):\n",
    "        state, _, _, action, next_state, reward, done, next_action \\\n",
    "                = self.trajectory[-8:]\n",
    "\n",
    "        # train actor\n",
    "        states = state[np.newaxis]\n",
    "        preds = self.critic_net.predict(states)\n",
    "        q = preds[0, action]\n",
    "        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)\n",
    "        with tf.GradientTape() as tape:\n",
    "            pi_tensor = self.actor_net(state_tensor)[0, action]\n",
    "            log_pi_tensor = tf.math.log(tf.clip_by_value(pi_tensor,\n",
    "                    1e-6, 1.))\n",
    "            loss_tensor = -self.discount * q * log_pi_tensor\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_net.variables)\n",
    "        self.actor_net.optimizer.apply_gradients(zip(\n",
    "                grad_tensors, self.actor_net.variables))\n",
    "\n",
    "        # train critic\n",
    "        next_q = self.critic_net.predict(\n",
    "                next_state[np.newaxis])[0, next_action]\n",
    "        preds[0, action] = reward + (1. - done) * self.gamma * next_q\n",
    "        self.critic_net.fit(states, preds, verbose=0)\n",
    "\n",
    "\n",
    "agent = QActorCriticAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21:54:58 [INFO] ==== train ====\n",
      "21:57:40 [DEBUG] train episode 0: reward = -429.00, steps = 430\n",
      "22:00:44 [DEBUG] train episode 1: reward = -500.00, steps = 500\n",
      "22:03:49 [DEBUG] train episode 2: reward = -500.00, steps = 500\n",
      "22:06:54 [DEBUG] train episode 3: reward = -500.00, steps = 500\n",
      "22:10:00 [DEBUG] train episode 4: reward = -500.00, steps = 500\n",
      "22:13:06 [DEBUG] train episode 5: reward = -500.00, steps = 500\n",
      "22:16:12 [DEBUG] train episode 6: reward = -500.00, steps = 500\n",
      "22:19:18 [DEBUG] train episode 7: reward = -500.00, steps = 500\n",
      "22:22:32 [DEBUG] train episode 8: reward = -500.00, steps = 500\n",
      "22:25:37 [DEBUG] train episode 9: reward = -464.00, steps = 465\n",
      "22:28:08 [DEBUG] train episode 10: reward = -388.00, steps = 389\n",
      "22:31:13 [DEBUG] train episode 11: reward = -500.00, steps = 500\n",
      "22:34:15 [DEBUG] train episode 12: reward = -500.00, steps = 500\n",
      "22:37:14 [DEBUG] train episode 13: reward = -500.00, steps = 500\n",
      "22:39:47 [DEBUG] train episode 14: reward = -408.00, steps = 409\n",
      "22:42:28 [DEBUG] train episode 15: reward = -427.00, steps = 428\n",
      "22:44:46 [DEBUG] train episode 16: reward = -366.00, steps = 367\n",
      "22:46:49 [DEBUG] train episode 17: reward = -320.00, steps = 321\n",
      "22:49:33 [DEBUG] train episode 18: reward = -411.00, steps = 412\n",
      "22:51:31 [DEBUG] train episode 19: reward = -306.00, steps = 307\n",
      "22:54:42 [DEBUG] train episode 20: reward = -500.00, steps = 500\n",
      "22:57:26 [DEBUG] train episode 21: reward = -441.00, steps = 442\n",
      "22:59:18 [DEBUG] train episode 22: reward = -308.00, steps = 309\n",
      "23:00:34 [DEBUG] train episode 23: reward = -204.00, steps = 205\n",
      "23:03:22 [DEBUG] train episode 24: reward = -455.00, steps = 456\n",
      "23:06:30 [DEBUG] train episode 25: reward = -500.00, steps = 500\n",
      "23:09:14 [DEBUG] train episode 26: reward = -429.00, steps = 430\n",
      "23:11:34 [DEBUG] train episode 27: reward = -383.00, steps = 384\n",
      "23:14:36 [DEBUG] train episode 28: reward = -500.00, steps = 500\n",
      "23:17:40 [DEBUG] train episode 29: reward = -499.00, steps = 500\n",
      "23:20:27 [DEBUG] train episode 30: reward = -458.00, steps = 459\n",
      "23:23:29 [DEBUG] train episode 31: reward = -500.00, steps = 500\n",
      "23:25:18 [DEBUG] train episode 32: reward = -300.00, steps = 301\n",
      "23:26:44 [DEBUG] train episode 33: reward = -232.00, steps = 233\n",
      "23:29:05 [DEBUG] train episode 34: reward = -369.00, steps = 370\n",
      "23:31:31 [DEBUG] train episode 35: reward = -379.00, steps = 380\n",
      "23:33:09 [DEBUG] train episode 36: reward = -260.00, steps = 261\n",
      "23:34:55 [DEBUG] train episode 37: reward = -283.00, steps = 284\n",
      "23:37:31 [DEBUG] train episode 38: reward = -415.00, steps = 416\n",
      "23:39:08 [DEBUG] train episode 39: reward = -261.00, steps = 262\n",
      "23:40:46 [DEBUG] train episode 40: reward = -252.00, steps = 253\n",
      "23:43:34 [DEBUG] train episode 41: reward = -413.00, steps = 414\n",
      "23:46:04 [DEBUG] train episode 42: reward = -361.00, steps = 362\n",
      "23:47:20 [DEBUG] train episode 43: reward = -206.00, steps = 207\n",
      "23:48:58 [DEBUG] train episode 44: reward = -264.00, steps = 265\n",
      "23:50:55 [DEBUG] train episode 45: reward = -315.00, steps = 316\n",
      "23:52:09 [DEBUG] train episode 46: reward = -186.00, steps = 187\n",
      "23:53:58 [DEBUG] train episode 47: reward = -265.00, steps = 266\n",
      "23:55:55 [DEBUG] train episode 48: reward = -276.00, steps = 277\n",
      "23:57:30 [DEBUG] train episode 49: reward = -217.00, steps = 218\n",
      "23:59:21 [DEBUG] train episode 50: reward = -329.00, steps = 330\n",
      "00:00:16 [DEBUG] train episode 51: reward = -170.00, steps = 171\n",
      "00:01:17 [DEBUG] train episode 52: reward = -190.00, steps = 191\n",
      "00:02:16 [DEBUG] train episode 53: reward = -186.00, steps = 187\n",
      "00:03:16 [DEBUG] train episode 54: reward = -185.00, steps = 186\n",
      "00:04:02 [DEBUG] train episode 55: reward = -139.00, steps = 140\n",
      "00:05:15 [DEBUG] train episode 56: reward = -208.00, steps = 209\n",
      "00:06:43 [DEBUG] train episode 57: reward = -269.00, steps = 270\n",
      "00:07:48 [DEBUG] train episode 58: reward = -204.00, steps = 205\n",
      "00:08:57 [DEBUG] train episode 59: reward = -217.00, steps = 218\n",
      "00:10:12 [DEBUG] train episode 60: reward = -230.00, steps = 231\n",
      "00:11:02 [DEBUG] train episode 61: reward = -158.00, steps = 159\n",
      "00:11:48 [DEBUG] train episode 62: reward = -141.00, steps = 142\n",
      "00:13:23 [DEBUG] train episode 63: reward = -299.00, steps = 300\n",
      "00:14:19 [DEBUG] train episode 64: reward = -174.00, steps = 175\n",
      "00:15:13 [DEBUG] train episode 65: reward = -167.00, steps = 168\n",
      "00:16:15 [DEBUG] train episode 66: reward = -195.00, steps = 196\n",
      "00:17:51 [DEBUG] train episode 67: reward = -299.00, steps = 300\n",
      "00:19:00 [DEBUG] train episode 68: reward = -205.00, steps = 206\n",
      "00:19:56 [DEBUG] train episode 69: reward = -156.00, steps = 157\n",
      "00:20:56 [DEBUG] train episode 70: reward = -182.00, steps = 183\n",
      "00:21:53 [DEBUG] train episode 71: reward = -178.00, steps = 179\n",
      "00:22:41 [DEBUG] train episode 72: reward = -148.00, steps = 149\n",
      "00:23:53 [DEBUG] train episode 73: reward = -206.00, steps = 207\n",
      "00:24:48 [DEBUG] train episode 74: reward = -162.00, steps = 163\n",
      "00:25:35 [DEBUG] train episode 75: reward = -138.00, steps = 139\n",
      "00:26:29 [DEBUG] train episode 76: reward = -150.00, steps = 151\n",
      "00:27:29 [DEBUG] train episode 77: reward = -164.00, steps = 165\n",
      "00:28:43 [DEBUG] train episode 78: reward = -192.00, steps = 193\n",
      "00:29:37 [DEBUG] train episode 79: reward = -143.00, steps = 144\n",
      "00:30:45 [DEBUG] train episode 80: reward = -176.00, steps = 177\n",
      "00:31:44 [DEBUG] train episode 81: reward = -163.00, steps = 164\n",
      "00:32:40 [DEBUG] train episode 82: reward = -148.00, steps = 149\n",
      "00:33:21 [DEBUG] train episode 83: reward = -121.00, steps = 122\n",
      "00:34:20 [DEBUG] train episode 84: reward = -178.00, steps = 179\n",
      "00:35:13 [DEBUG] train episode 85: reward = -162.00, steps = 163\n",
      "00:36:05 [DEBUG] train episode 86: reward = -145.00, steps = 146\n",
      "08:41:02 [DEBUG] train episode 87: reward = -153.00, steps = 154\n",
      "08:42:01 [DEBUG] train episode 88: reward = -168.00, steps = 169\n",
      "08:42:53 [DEBUG] train episode 89: reward = -163.00, steps = 164\n",
      "08:43:37 [DEBUG] train episode 90: reward = -137.00, steps = 138\n",
      "08:44:26 [DEBUG] train episode 91: reward = -138.00, steps = 139\n",
      "08:45:19 [DEBUG] train episode 92: reward = -149.00, steps = 150\n",
      "08:46:25 [DEBUG] train episode 93: reward = -187.00, steps = 188\n",
      "08:47:04 [DEBUG] train episode 94: reward = -111.00, steps = 112\n",
      "08:48:03 [DEBUG] train episode 95: reward = -172.00, steps = 173\n",
      "08:49:17 [DEBUG] train episode 96: reward = -232.00, steps = 233\n",
      "08:49:53 [DEBUG] train episode 97: reward = -112.00, steps = 113\n",
      "08:50:51 [DEBUG] train episode 98: reward = -182.00, steps = 183\n",
      "08:51:37 [DEBUG] train episode 99: reward = -146.00, steps = 147\n",
      "08:52:33 [DEBUG] train episode 100: reward = -177.00, steps = 178\n",
      "08:53:07 [DEBUG] train episode 101: reward = -104.00, steps = 105\n",
      "08:54:01 [DEBUG] train episode 102: reward = -169.00, steps = 170\n",
      "08:54:33 [DEBUG] train episode 103: reward = -102.00, steps = 103\n",
      "08:55:33 [DEBUG] train episode 104: reward = -191.00, steps = 192\n",
      "08:56:22 [DEBUG] train episode 105: reward = -153.00, steps = 154\n",
      "08:57:27 [DEBUG] train episode 106: reward = -204.00, steps = 205\n",
      "08:58:12 [DEBUG] train episode 107: reward = -142.00, steps = 143\n",
      "08:59:00 [DEBUG] train episode 108: reward = -152.00, steps = 153\n",
      "08:59:36 [DEBUG] train episode 109: reward = -111.00, steps = 112\n",
      "09:00:16 [DEBUG] train episode 110: reward = -127.00, steps = 128\n",
      "09:00:53 [DEBUG] train episode 111: reward = -116.00, steps = 117\n",
      "09:01:34 [DEBUG] train episode 112: reward = -129.00, steps = 130\n",
      "09:02:17 [DEBUG] train episode 113: reward = -133.00, steps = 134\n",
      "09:03:18 [DEBUG] train episode 114: reward = -193.00, steps = 194\n",
      "09:04:01 [DEBUG] train episode 115: reward = -117.00, steps = 118\n",
      "09:05:04 [DEBUG] train episode 116: reward = -176.00, steps = 177\n",
      "09:05:55 [DEBUG] train episode 117: reward = -142.00, steps = 143\n",
      "09:06:44 [DEBUG] train episode 118: reward = -140.00, steps = 141\n",
      "09:07:29 [DEBUG] train episode 119: reward = -121.00, steps = 122\n",
      "09:08:20 [DEBUG] train episode 120: reward = -141.00, steps = 142\n",
      "09:08:58 [DEBUG] train episode 121: reward = -110.00, steps = 111\n",
      "09:09:48 [DEBUG] train episode 122: reward = -151.00, steps = 152\n",
      "09:10:46 [DEBUG] train episode 123: reward = -164.00, steps = 165\n",
      "09:11:20 [DEBUG] train episode 124: reward = -98.00, steps = 99\n",
      "09:12:00 [DEBUG] train episode 125: reward = -121.00, steps = 122\n",
      "09:12:40 [DEBUG] train episode 126: reward = -123.00, steps = 124\n",
      "09:13:27 [DEBUG] train episode 127: reward = -143.00, steps = 144\n",
      "09:14:10 [DEBUG] train episode 128: reward = -127.00, steps = 128\n",
      "09:15:03 [DEBUG] train episode 129: reward = -157.00, steps = 158\n",
      "09:16:02 [DEBUG] train episode 130: reward = -181.00, steps = 182\n",
      "09:16:46 [DEBUG] train episode 131: reward = -122.00, steps = 123\n",
      "09:17:32 [DEBUG] train episode 132: reward = -137.00, steps = 138\n",
      "09:18:20 [DEBUG] train episode 133: reward = -132.00, steps = 133\n",
      "09:19:03 [DEBUG] train episode 134: reward = -118.00, steps = 119\n",
      "09:19:44 [DEBUG] train episode 135: reward = -117.00, steps = 118\n",
      "09:20:36 [DEBUG] train episode 136: reward = -164.00, steps = 165\n",
      "09:21:14 [DEBUG] train episode 137: reward = -118.00, steps = 119\n",
      "09:22:56 [DEBUG] train episode 138: reward = -300.00, steps = 301\n",
      "09:23:57 [DEBUG] train episode 139: reward = -169.00, steps = 170\n",
      "09:24:44 [DEBUG] train episode 140: reward = -138.00, steps = 139\n",
      "09:27:10 [DEBUG] train episode 141: reward = -417.00, steps = 418\n",
      "09:28:05 [DEBUG] train episode 142: reward = -153.00, steps = 154\n",
      "09:28:52 [DEBUG] train episode 143: reward = -137.00, steps = 138\n",
      "09:29:40 [DEBUG] train episode 144: reward = -140.00, steps = 141\n",
      "09:30:24 [DEBUG] train episode 145: reward = -124.00, steps = 125\n",
      "09:31:18 [DEBUG] train episode 146: reward = -159.00, steps = 160\n",
      "09:32:14 [DEBUG] train episode 147: reward = -155.00, steps = 156\n",
      "09:33:12 [DEBUG] train episode 148: reward = -170.00, steps = 171\n",
      "09:33:58 [DEBUG] train episode 149: reward = -132.00, steps = 133\n",
      "09:34:33 [DEBUG] train episode 150: reward = -102.00, steps = 103\n",
      "09:35:18 [DEBUG] train episode 151: reward = -129.00, steps = 130\n",
      "09:35:56 [DEBUG] train episode 152: reward = -106.00, steps = 107\n",
      "09:36:42 [DEBUG] train episode 153: reward = -122.00, steps = 123\n",
      "09:37:41 [DEBUG] train episode 154: reward = -160.00, steps = 161\n",
      "09:38:38 [DEBUG] train episode 155: reward = -163.00, steps = 164\n",
      "09:39:21 [DEBUG] train episode 156: reward = -126.00, steps = 127\n",
      "09:40:09 [DEBUG] train episode 157: reward = -139.00, steps = 140\n",
      "09:40:48 [DEBUG] train episode 158: reward = -111.00, steps = 112\n",
      "09:41:43 [DEBUG] train episode 159: reward = -159.00, steps = 160\n",
      "09:42:29 [DEBUG] train episode 160: reward = -136.00, steps = 137\n",
      "09:43:24 [DEBUG] train episode 161: reward = -154.00, steps = 155\n",
      "09:44:06 [DEBUG] train episode 162: reward = -109.00, steps = 110\n",
      "09:44:54 [DEBUG] train episode 163: reward = -129.00, steps = 130\n",
      "09:45:59 [DEBUG] train episode 164: reward = -173.00, steps = 174\n",
      "09:47:14 [DEBUG] train episode 165: reward = -203.00, steps = 204\n",
      "09:47:56 [DEBUG] train episode 166: reward = -111.00, steps = 112\n",
      "09:48:47 [DEBUG] train episode 167: reward = -135.00, steps = 136\n",
      "09:49:41 [DEBUG] train episode 168: reward = -146.00, steps = 147\n",
      "09:50:25 [DEBUG] train episode 169: reward = -119.00, steps = 120\n",
      "09:51:20 [DEBUG] train episode 170: reward = -146.00, steps = 147\n",
      "09:52:19 [DEBUG] train episode 171: reward = -160.00, steps = 161\n",
      "09:53:06 [DEBUG] train episode 172: reward = -129.00, steps = 130\n",
      "09:54:04 [DEBUG] train episode 173: reward = -157.00, steps = 158\n",
      "09:54:51 [DEBUG] train episode 174: reward = -125.00, steps = 126\n",
      "09:55:30 [DEBUG] train episode 175: reward = -104.00, steps = 105\n",
      "09:56:06 [DEBUG] train episode 176: reward = -97.00, steps = 98\n",
      "09:56:53 [DEBUG] train episode 177: reward = -124.00, steps = 125\n",
      "09:57:43 [DEBUG] train episode 178: reward = -137.00, steps = 138\n",
      "09:58:39 [DEBUG] train episode 179: reward = -148.00, steps = 149\n",
      "09:59:17 [DEBUG] train episode 180: reward = -106.00, steps = 107\n",
      "10:00:09 [DEBUG] train episode 181: reward = -139.00, steps = 140\n",
      "10:00:47 [DEBUG] train episode 182: reward = -103.00, steps = 104\n",
      "10:01:26 [DEBUG] train episode 183: reward = -107.00, steps = 108\n",
      "10:01:26 [INFO] ==== test ====\n",
      "10:01:39 [DEBUG] test episode 0: reward = -136.00, steps = 137\n",
      "10:01:51 [DEBUG] test episode 1: reward = -129.00, steps = 130\n",
      "10:02:00 [DEBUG] test episode 2: reward = -102.00, steps = 103\n",
      "10:02:10 [DEBUG] test episode 3: reward = -104.00, steps = 105\n",
      "10:02:20 [DEBUG] test episode 4: reward = -113.00, steps = 114\n",
      "10:02:29 [DEBUG] test episode 5: reward = -104.00, steps = 105\n",
      "10:02:42 [DEBUG] test episode 6: reward = -136.00, steps = 137\n",
      "10:02:56 [DEBUG] test episode 7: reward = -164.00, steps = 165\n",
      "10:03:08 [DEBUG] test episode 8: reward = -128.00, steps = 129\n",
      "10:03:21 [DEBUG] test episode 9: reward = -144.00, steps = 145\n",
      "10:03:32 [DEBUG] test episode 10: reward = -127.00, steps = 128\n",
      "10:03:43 [DEBUG] test episode 11: reward = -117.00, steps = 118\n",
      "10:03:55 [DEBUG] test episode 12: reward = -131.00, steps = 132\n",
      "10:04:15 [DEBUG] test episode 13: reward = -231.00, steps = 232\n",
      "10:04:30 [DEBUG] test episode 14: reward = -163.00, steps = 164\n",
      "10:04:42 [DEBUG] test episode 15: reward = -126.00, steps = 127\n",
      "10:04:52 [DEBUG] test episode 16: reward = -120.00, steps = 121\n",
      "10:05:01 [DEBUG] test episode 17: reward = -101.00, steps = 102\n",
      "10:05:11 [DEBUG] test episode 18: reward = -110.00, steps = 111\n",
      "10:05:24 [DEBUG] test episode 19: reward = -142.00, steps = 143\n",
      "10:05:34 [DEBUG] test episode 20: reward = -121.00, steps = 122\n",
      "10:05:49 [DEBUG] test episode 21: reward = -171.00, steps = 172\n",
      "10:06:01 [DEBUG] test episode 22: reward = -140.00, steps = 141\n",
      "10:06:14 [DEBUG] test episode 23: reward = -141.00, steps = 142\n",
      "10:06:25 [DEBUG] test episode 24: reward = -125.00, steps = 126\n",
      "10:06:36 [DEBUG] test episode 25: reward = -126.00, steps = 127\n",
      "10:06:49 [DEBUG] test episode 26: reward = -144.00, steps = 145\n",
      "10:06:58 [DEBUG] test episode 27: reward = -99.00, steps = 100\n",
      "10:07:07 [DEBUG] test episode 28: reward = -101.00, steps = 102\n",
      "10:07:18 [DEBUG] test episode 29: reward = -122.00, steps = 123\n",
      "10:07:30 [DEBUG] test episode 30: reward = -129.00, steps = 130\n",
      "10:07:40 [DEBUG] test episode 31: reward = -109.00, steps = 110\n",
      "10:07:54 [DEBUG] test episode 32: reward = -160.00, steps = 161\n",
      "10:08:05 [DEBUG] test episode 33: reward = -124.00, steps = 125\n",
      "10:08:15 [DEBUG] test episode 34: reward = -102.00, steps = 103\n",
      "10:08:24 [DEBUG] test episode 35: reward = -107.00, steps = 108\n",
      "10:08:34 [DEBUG] test episode 36: reward = -107.00, steps = 108\n",
      "10:08:44 [DEBUG] test episode 37: reward = -107.00, steps = 108\n",
      "10:08:57 [DEBUG] test episode 38: reward = -133.00, steps = 134\n",
      "10:09:08 [DEBUG] test episode 39: reward = -127.00, steps = 128\n",
      "10:09:19 [DEBUG] test episode 40: reward = -121.00, steps = 122\n",
      "10:09:30 [DEBUG] test episode 41: reward = -124.00, steps = 125\n",
      "10:09:41 [DEBUG] test episode 42: reward = -126.00, steps = 127\n",
      "10:09:53 [DEBUG] test episode 43: reward = -128.00, steps = 129\n",
      "10:10:01 [DEBUG] test episode 44: reward = -89.00, steps = 90\n",
      "10:10:11 [DEBUG] test episode 45: reward = -116.00, steps = 117\n",
      "10:10:23 [DEBUG] test episode 46: reward = -134.00, steps = 135\n",
      "10:10:36 [DEBUG] test episode 47: reward = -142.00, steps = 143\n",
      "10:10:47 [DEBUG] test episode 48: reward = -123.00, steps = 124\n",
      "10:11:03 [DEBUG] test episode 49: reward = -165.00, steps = 166\n",
      "10:11:12 [DEBUG] test episode 50: reward = -105.00, steps = 106\n",
      "10:11:30 [DEBUG] test episode 51: reward = -194.00, steps = 195\n",
      "10:11:39 [DEBUG] test episode 52: reward = -108.00, steps = 109\n",
      "10:11:53 [DEBUG] test episode 53: reward = -140.00, steps = 141\n",
      "10:12:02 [DEBUG] test episode 54: reward = -105.00, steps = 106\n",
      "10:12:12 [DEBUG] test episode 55: reward = -108.00, steps = 109\n",
      "10:12:24 [DEBUG] test episode 56: reward = -133.00, steps = 134\n",
      "10:12:36 [DEBUG] test episode 57: reward = -132.00, steps = 133\n",
      "10:12:49 [DEBUG] test episode 58: reward = -148.00, steps = 149\n",
      "10:13:08 [DEBUG] test episode 59: reward = -222.00, steps = 223\n",
      "10:13:18 [DEBUG] test episode 60: reward = -108.00, steps = 109\n",
      "10:13:31 [DEBUG] test episode 61: reward = -143.00, steps = 144\n",
      "10:13:43 [DEBUG] test episode 62: reward = -130.00, steps = 131\n",
      "10:13:52 [DEBUG] test episode 63: reward = -95.00, steps = 96\n",
      "10:14:05 [DEBUG] test episode 64: reward = -147.00, steps = 148\n",
      "10:14:17 [DEBUG] test episode 65: reward = -122.00, steps = 123\n",
      "10:14:26 [DEBUG] test episode 66: reward = -96.00, steps = 97\n",
      "10:14:37 [DEBUG] test episode 67: reward = -123.00, steps = 124\n",
      "10:14:48 [DEBUG] test episode 68: reward = -121.00, steps = 122\n",
      "10:14:58 [DEBUG] test episode 69: reward = -109.00, steps = 110\n",
      "10:15:07 [DEBUG] test episode 70: reward = -100.00, steps = 101\n",
      "10:15:17 [DEBUG] test episode 71: reward = -109.00, steps = 110\n",
      "10:15:28 [DEBUG] test episode 72: reward = -115.00, steps = 116\n",
      "10:15:39 [DEBUG] test episode 73: reward = -114.00, steps = 115\n",
      "10:15:50 [DEBUG] test episode 74: reward = -111.00, steps = 112\n",
      "10:16:00 [DEBUG] test episode 75: reward = -108.00, steps = 109\n",
      "10:16:13 [DEBUG] test episode 76: reward = -138.00, steps = 139\n",
      "10:16:23 [DEBUG] test episode 77: reward = -106.00, steps = 107\n",
      "10:16:36 [DEBUG] test episode 78: reward = -143.00, steps = 144\n",
      "10:16:54 [DEBUG] test episode 79: reward = -202.00, steps = 203\n",
      "10:17:06 [DEBUG] test episode 80: reward = -125.00, steps = 126\n",
      "10:17:28 [DEBUG] test episode 81: reward = -238.00, steps = 239\n",
      "10:17:41 [DEBUG] test episode 82: reward = -149.00, steps = 150\n",
      "10:17:53 [DEBUG] test episode 83: reward = -131.00, steps = 132\n",
      "10:18:06 [DEBUG] test episode 84: reward = -143.00, steps = 144\n",
      "10:18:17 [DEBUG] test episode 85: reward = -123.00, steps = 124\n",
      "10:18:30 [DEBUG] test episode 86: reward = -142.00, steps = 143\n",
      "10:18:40 [DEBUG] test episode 87: reward = -104.00, steps = 105\n",
      "10:18:50 [DEBUG] test episode 88: reward = -106.00, steps = 107\n",
      "10:19:02 [DEBUG] test episode 89: reward = -133.00, steps = 134\n",
      "10:19:11 [DEBUG] test episode 90: reward = -107.00, steps = 108\n",
      "10:19:26 [DEBUG] test episode 91: reward = -159.00, steps = 160\n",
      "10:19:37 [DEBUG] test episode 92: reward = -114.00, steps = 115\n",
      "10:19:49 [DEBUG] test episode 93: reward = -141.00, steps = 142\n",
      "10:20:03 [DEBUG] test episode 94: reward = -167.00, steps = 168\n",
      "10:20:48 [DEBUG] test episode 95: reward = -500.00, steps = 500\n",
      "10:21:02 [DEBUG] test episode 96: reward = -154.00, steps = 155\n",
      "10:21:14 [DEBUG] test episode 97: reward = -127.00, steps = 128\n",
      "10:21:23 [DEBUG] test episode 98: reward = -102.00, steps = 103\n",
      "10:21:31 [DEBUG] test episode 99: reward = -94.00, steps = 95\n",
      "10:21:31 [INFO] average episode reward = -132.85 ± 45.99\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOy9eZhcV30m/J671db7IqnVLVmLZcnybgtjMDaLMTiEYAhJYCYJyQcTjxmSkIdJyOcww8wXxplkmPlIIIOJQ5jECeCwxDFht2NsYyzbSMjyKtnau6WW1HtXdVXdusuZP+49555769bWVV296LzP04+6b223W1Xvee/7e3+/QyilkJCQkJC4sKAs9wlISEhISLQfkvwlJCQkLkBI8peQkJC4ACHJX0JCQuIChCR/CQkJiQsQ2nKfQL0YGBigW7ZsWe7TkJCQkFhV2L9//ySldDB6fNWQ/5YtW7Bv377lPg0JCQmJVQVCyMm449L2kZCQkLgAIclfQkJC4gKEJH8JCQmJCxCS/CUkJCQuQEjyl5CQkLgAIclfQkJC4gKEJH8JCQmJCxBNkT8h5JcJIS8SQlxCyJ7IbXcRQo4QQg4TQt4uHL+OEPK8f9tnCSGkmXOQkJCQWC1wXYqv7xvFXN5a7lNpWvm/AOAXATwuHiSE7AbwfgCXAbgNwOcJIap/8z0A7gCww/+6rclzkJCQaDH+5onj+NUvPrXcp7Hm8MOXzuIPvvEc/vnZ08t9Ks2RP6X0ZUrp4ZibbgdwP6XUpJQeB3AEwPWEkCEAXZTSvdTbReY+AO9u5hwkJCRaj+fHZvHk0SkULWe5T2XNgFKKzz1yBAAwOp1f5rNZOs9/GMCo8POYf2zY/z56PBaEkDsIIfsIIfsmJiaW5EQlJFYK/vGnp3Dbnz9e+45tQM50QClwagWQVC389MQ0rvnjH2IqZy73qVTFo4cn8OKZeQDA2Eyh6n1dl+KvHz+Gs3PFJTufmuRPCHmYEPJCzNft1R4Wc4xWOR4LSum9lNI9lNI9g4Nlc4kkJFY9fuNLz+DTPzgEAHh5PItDZ7Nw3OXfWnXBtAEAxyYWlvlMauPAqRnM5C0cPpcF4BHnStye9ks/OY7hnhTecPEAxmbLF1VKg/M+OpHD3d99Gb/5f55Bzv+/aDVqkj+l9K2U0stjvh6s8rAxAJuEn0cAnPGPj8Qcl5C4IHH4bBaHz+YAgH/ITXv5rZaFkncuxydXPvmfmfXU8dhMAabt4Po/+Vd8fd9YjUe1Hy+cnsPNlwxg60AmVvl/6+AZvObuh2HaDs5nvauYQ2ez+J2v/Ay247b8fJbK9vkWgPcTQhKEkK3wCrvPUErHAWQJITf4KZ8PAKi2iEhIrGkslGzkTC/5wdR2odRe8ndcipIdJhe2EB2fzLX1XBYDZo2MzRRwaiqPyZyJR185v+jnOz9fxP/+0RG4LbwCm8qZmMlb2D7YgZHeFGbzFrLFcOLn5fEsJnMlnJszMeGT/wdv3IojEzlM5kotOxeGZqOe7yGEjAF4HYDvEEJ+AACU0hcBfA3ASwC+D+AjlFL2jv4wgC/CKwIfBfC9Zs5BQmIl4ehEDl964njomOvSWEKnlCJfcrBgercxwi3UKLK+eGYOf/fkidacMIDPPfIq3vWXT4SOLZirR/mPz3kqemwmz8/34Ohc1cdQSnHPo0dxaqrcfvnu8+P49A8OcxtJxCOHzuGhl841fI5HznuL6MXrOjDSm/bPN6z+pxc8wj+XLXLy/71bd+D7H70ZG7qTDb9mLTSb9nmAUjpCKU1QStdTSt8u3HY3pXQ7pXQnpfR7wvF9vm20nVL623QlmnMSEovEPx84jT/+9kuhlMyXnzmFN376R2U+tGm7cFzKiZb9Wythc/8zo/gv33oRc4XWZMVfPDOPV8/nQkqXLUjHJ1d+wfeMoPwZ+Z+eLeB8tnKx9Ny8iT/7/iF862B55HLaz+C/EkP+X3j0GP7XD+MCjtXxqk/+O9Z3YqQ3xc839LoLJf/cipjImUhoCjoTGjKJpdl2RXb4Ski0EFEVD3iK9HzWRNEKWyvsaiDLyd/xj1f3d6d8hfj8mKduJ5tMuZydK8JxKeZ9G8J1KXKmjZSuYjJn8uMrESXb5b//2HQeJ6aCK5Vq6v+0X3CdjWm2mvFJ+NVz5ZZXwXJwYmqh4YLykfM5pA0VG7uTAvmHF9aA/D3bZ7AzgaXsgZXkLyHRQhQsj8hzxYD8mZ8eTW2woipT/Oz2Yo2C75Tv/x4cm8XB0Vm85u6H8eTRybL77T85gy/++FjNcx73lfOUTz55/8rjso1dAIATi7R+Ts8W8NePH8Nd//Q8v5qZWSjBamHx8tx8EZQCQ91JnJ0v4vDZLC7b2AVVITg4OlvxcUx1z8ZcPc3kvb9DnO1TsBwULZcXZOvF0Ykctg92gBCCvoyBlK5WVf7ns0UMdiYaeo1GIclfQqKFYOo9G0P++VKY/PO+8s+XHM/+KdVX8GUkceDULL7z/DgoBR55ubzA+cCBMfzPGhaFqJzZ87LF6PLhbgCL8/1Ltovb//IJ3P3dl/HVZ07hhdNzoJTi1s883tJ6xZlZj0Bfs6UPLgWeG5vD7qEu7FzfiYNjlcn/tP+4OOXPjr0aQ/5sERMXxAcOjOGfD1Tv2D1yPoeL13UAAAghGOlNlSn/KdH2yZoY7JDkLyGx5PjqM6fwLwebTx3nuZUTkEpF5S/8vFCyg7RPDc+fkfSzo7O8+PiTo1Pl52J6KjVOaX/uX1/Fj1+dwLn5wBdnTVLsPHcPdYEQrzkpmgaK4tx8MVSreP70HCZzJXzoDVsBADN5CwslB5M5s2HVXA3squU1W/sAALZLsWUgg6s39+Dg6GzFxM5pX3XPFcpTNOzve3I6X1Z/YdbdSaFQ/FePHcO9j1e+wsoWLYzPFTn5A8CmvnRI+VuOywUDJ3+p/CUklh5/9dhRfOXpU00/D1P3IdvHYcrfidw3+Hl2wYLleERVreDruhQz+RJ60zomcyaOTy5gc18aL4/Pl3W48oWoGF50ipaDzzz8Cv7hqZM4K5J/RPn3ZQz8ynWb8MCB0/j5z/646jCyX/jcE/jzh1/lPz9zfBoA8M4rh7zfL1/iXrrtNOaXzxUsnn6J4oyf9Ll+Sx8/tm0ggyuHuzFftLnCj6K68i+hM6GB0iClw8D+b477tQVKKcZmChidqVwYP+o3yonk7yn/4NzY3wbwLKmZvIV1na1P+IiQ5C9xwcN1KU7PFkJqfbGII1zTV4sLVZT/OSGZUo38ZwsWXAq8aec6fuyun9sFANh7LKz+F2IWIsAjNJcCr5zLceUMANN+LYEp/0xCw5/90pX4zPuuwqvnc7F1BcBTreezJvYKtz9zfArbBzOc8GbzFida223M8/+jB57Hh/7up7G3jc8W0Z3SsW0wA8WvjW4dzPBopJj4sR2Xk/npqp6/hT1begGUJ37Y/81Jn/znChZypo1s0a6YvhJjngyb+9KhRY0tvEPdSb4oSOUvIbHEOJctwnJomUJeDLjyN8uVP6sHBPcNfhbtl2qeP8uC33jxAAxVwRXD3bh193p0JDQ8GbF+2PNE0zqHznqEdnJqASd971pXiaD8vcd1+BHD2y4bAiFBAfSP/+UlPHo4qDGwxeXFM/MoWl79Yt/JGVy/tR8dCQ2aQjBbKPFCqtWA8qeU4ulj0xWLzuNzBQx1J6GrCoa6vRTNlv4MBny/fCIbKOrvvXAWt37mMRybyHHlP5e3QsmdouWgYDm4elMvDFUJFX0tx4Xt20gn/AisqN6jHj7D6HQehACb/Hw/AOzxr1Se8hdspvx3D3Xx+0jyl5BYYrAP8HwLcvOM0EPk7/vlC6X4tA+A0ACvglVZGbNOz6HuJP7g7TvxsbddAk1V8NqtfXjs8EToqqGS7XNo3Bsu5lLgiSOT6Eho2NiT4l436zjuSHrknzJUbO5L49VzOUwvlPClnxzHt58b58/Hnt92KZ4bm8Ohs/PIFm28dmsfCCHoSeuYyVtcZTcyqmB8rujHTe3YK6Izs0UM+Sp/uDeF4Z4UkrrKiVOMwZ6azoNS4F8OjiNfcjDYmUDJcUM1FrZADXYmsG0wE4p7stdXFYKTftxTJHxxIfibJ47jvr0nAHgLe38mAUML6PaK4W50JjV+NcUW3ksl+UtItA/sA5wt2k0PBItT25z8I7ZPXrgSEIug1Qq+jKD7MgZ+6+ZteLNv//zaDRfh9GwBH//Gc/x3iLsKATwF3+kT+/6TM9jQnURfxuD9Azn/vDIJlT/mkvWdOHwuy+OTrKsWCBe395+c4X7/9X4RtjulYy5vYdYn1kYG1z0rxDXjfP/xuQKGejzFf8dN2/B7b90BwPv7AGHyZ4//+n5v4DCLss4Ivv/Mgvd9b1rHSG86ZIux/5etAxm/eF3C6LSo/IPv//Gnp3D/M97rnJ0vYkN3mMhVheCGbf34yRFP+bP/111Dnfw+kvwlJJYYY/4H2HZpWSNWo4jz2dmgtmjBV1T+ou1jViF/phD7fXJjePOudfj4bTvxrYNncN/ek6HXi5shc8uuddBVAtulGOpOoj9j8P4Btkh1CJ2ll6zvwInJBTxzwiP28dngfMUri73HpnD/M6PYOpDBRp+Ue9IGZvIlTqxWA+QvZvWjKaGi5WAmb2Goy1P+b929Hr+8x5snqasKetI6/52AYCFgJM3Iny1K4vc9aQO9aT1UiGW1m10bPII+ObWAsZm814VrqKGrgPHZIreWzs4VsaGrvHj7+u39ODWdx+h0PiD/DYHyH+gwyh7TSkjyl7jgISq2KFE2AkdYPES1bVaIeoqLwdkYhRkHVpTtzZQTw4ffuB3DPSkcODUTen6RnCdzJiZzJi4f7sa2Aa8AuaErif5MIpTzVwiQ0sPK33YpvvWsF4c9M1fgVxjs+Xeu78Tjr0zg8LksPvnO3fyxvWkds3mLWyqN2D7Pjs4iY3jnMREZ18DIvJJCHuhIxCp/hss2en0Mc3kLo9N5nJhcwHSe/X119GW8RYv9nsz2YdbMsYkFjM0UMNKXxkhvEN3MFi1kTZsXg8/NF7E+hvxvvHgAALD36BSmF0roSevY2OPdrzulI6GpZY9pJST5S6w5PHlksqEdqMTZ6vN1FH1zpo33fP4n+O2v/AyPvRJsMiSSdjYu6hmT9un01XXI9qlR8O1KatDV8o8u89eZfcVsH3FBO+wXe3dt6MKO9R75D3Un0dcREF3OtJExtNBogR3rPLV7erYATSEoWi5P77AawZt2entuvG/PJrx5V5BG6k4ZmM2XuKqOFnyfHZ0NqW/Xpbjz7/fjnkeP4vnTczzZFFX+rP4xUKEZaqDDCJH/ZM7Ea/wUT0pXsXUgA8BL/Py///QcfvurP+MWUF/aQG/GgGkHNQH27451HejLGPjJ0UmP/HtToeimeBV3bCKHmbwVq/x3rOvAYGcCTxyZxHS+hL6MgbShoTOpLbnlA0jyl2ghDp2dxy/d82SZt91OnJkt4N9+8Wn8TWSyZjWMThe4B17PHJuXx+dx4NQsHjl0Hr/5f57hql3s4I0b77AQIfVCyUF3WoeuEv4cPWm9qvKfWiihv0rnZ2dSQ9a0YdoumLuSFf4/XjjtzbvZNdSJS9Z7hL6hO4X+jAHLoZgves1m0WFi2wYzUP0s5et9xcoy9myhe99rNuF33nIx/tM7Lw09tjetY7ZgcWJ1hKin5bj4lb/ai7/3rSrAy9B//8Wz+LPvH0K+5ODNu9ZBIcD5+TD5s76GgarKP1hUJrImdg91Yce6Dgz3ptCb9q6eZvMWXj2Xw+GzWUzMs/8Hz/YBAj+eXdVlEhresmsdHjl0HqMzeYz0prymrek8KKWhOsH+kzP+37ic/AkhuHnHIB57ZQIT8yb6/PNZ35Vc8u5eQJK/RAvx0xMz2HdypmrDi4i/ePhV/PfvvtzSc2A7Tz12uL5tPx2X4sxsgUfs6ol7sjHAn3nf1aAUePhlr8tWLOBmY2yfspx/yVPYHQmNE/5ARyK27nDg1Az3hqN+v4iOhKf8RUspW7ThuhR/8t2X8ekfHMbF6zow0JHg5D/kF3wBj1Bzps2TPgxJXcVF/V5U8ecu3wAg8P3Z32y4N4X/+Lad6Ezqocf2pHXkS8EGJbbg+Z/PmijZbmhhZD7/u67aiO6Ujtdv78dAR6LMtmGqvpI3PtCRwKT/GNN2MF+0MdCRwH9912X4o3fsQo9P7qdnvcF7lkPx9PFpZAwVhqbwxYHVKtj/UVJXcOvu9fzvPNKbxkhvClnTxnzBrpv8AeDW3esxV7Cw/9QM/z/4xDsu5YXrpcTSzAqVuCAxG+kQrYVv/GwU3Sm99h0bAJvquP/UDOYKVuj5f3JkEv//Q6/gq791A4/dnZsvwnYpLh3qwtPHp+vy/EdnvNz2m3YO4qL+NB566Rx+7YaLOOGmDTX0PJWUf77kIJ1QkSlpmMlb0FWCrqQWa1n97v0HsLkvjalciZNwHLqSGg6bVuj/IFu08cyJadz7+DG855ph/Kef95T5m3cN4q6f24XXX9yPvUeD1EnOdGLHCO9c34kzswW88RLP3mGJn/miBUNVKnrUPT6Jsqy+OG7irP8crpCyOjg6i7Sh4jPvuxoEgKIQDHYmykY012P7ZE0vIsoK5YOdCe61A0BCU0LTP392aoZ31jIyZrWKIid/FTftGEBCU2DaLkZ6U7wuMDqT51dxukrwM0b+MbYPANx8SfA8/f4iJlpmS4lmN3P5ZULIi4QQlxCyRzi+hRBSIIQ86399QbjtOkLI84SQI4SQz5KlnFkq0VbMcA+4tt8+m/dico22+tcC67x0XIonj4Q7UvedmMH+kzOh4irzaeOU/x9+4zl86tsvlb3G6HQBG7qSSGgqbr10PfYenULOtPlEz/Vdyficv+l58Q8+exqm7Xj2iq/8Ac9OSBlqrO0zu2Bh79EpjM3kOUnEoSOpIVu0I/UHiw9A+91bdnDbKKGp+Pdv3I6EpqI/4x2bWihhwbTRkSgn8t+9ZQf+/H3XYENXErpK+Bz9XNHmtlkcmMJm5yT+nzOVLMY/nx2dxRXD3VAVAsW3mtZ1Jso8/4msic6EhqQev+iwRWEyZ/IrgOhC0ZPWQ3FSy6Gc9HurkH/a0HDTDm8R8Tz/YIOW8bkiBjoMbOxJ8b/R+grKP21oeIO/GLErjXahWdvnBQC/CODxmNuOUkqv9r/uFI7fA+AOeFs77gBwW5PnILFCwIp20cJmHF447TUalZoc7/ud58ZDM2dOTOWxbTCDzqQWKsYCwKw/xEscpcDieSzBwRq9HJfiX547g/v2nihTnKPTed6teevu9Sg5Lh47PME7Ywc7E8gJPQNBh6+NF8/M46P3P4sfvHjOU/6GGpC/oSGlq2UFX9elyJVsuNS7euiravtoyBXtkPLPFW1OnOsq+ONsQZn2yT9jlJP5pUNduO3yDVAUgvVdSYzPBp5/NfKPkppo+5yNkL9pO3hpfB5Xb+oJPWZdZzKm4GtWXQgZ0U/lStwyihZSe1JGaJAdECxW7LwDzz8gfwD4lT2bsLE7ia0DGWz2r8ZeOZfF2bkCNnQnMexHXdOGygv7cbh193oAqPr/uhRodievlymldW9rQwgZAtBFKd3r7+B1H4B3N3MOEisHTCFFI41xeN4vPDaj/MfnCvjIV36Gf342GKd7cmoB2wc78IaLB/Do4YlQ0xabvSIq/1fP56ApBDvWd0AhgfI/OpFDvuTAcii++vRo6HVHZ/LY1Od92K+7qBe9aR2PHDrPbZ/1XUneM2A7Lie2hZLNkyBnZgue5y/s1NThq9joPP+85UDsPevLVCv46rD94W8AePrn3HwRHVV2hWLE49k+dijjH4eN3YGqzRatshqBiKi1J872Ycqf2T4vj2dhObSc/LsSmMqZoSuEyZxZ0fIBgkIwi7eKx/i5+UQ/3JPCtRd5r8lIvzulg5Bg9AKrxbAI7Nsu24An77oFaUNDV1LHrg2d+OmJaYz7uX5G/hu6klU3Zbl193qM9KZwhT9Cu11YyoLvVkLIAULIY4SQm/xjwwDGhPuM+cdiQQi5gxCyjxCyb2KivgKexPKB2T7RZqY4sNRJMxt78OFc+WAHqpNTeWwdyODmSwZxdr7IJyoC4FcIYhRv/8kZXDbcjaSuojOpc6+eFR23DWbw5adP8vM0bQdn54vY1Od9sDVVwY51nRibyfO0z3qfYLKmFbqyyZsOV6Bn54rIm77yTzLbR/XIP/L3Y8khNsagWvMPe65zfjJmfWcS2aKF81mzouoHPDXbkdBwbr7o2T5VyBzwCpjjQtqnM1G5dsOUNIO44EeVP/u7XxUh/8HOBFwa7GIGeJ5/VfLvCLp82d89Wizv8RemLQNpnvtnKR9VIehJ6fx9LRZ843D91j7sPzmD07Oe8mdWUK39d/s7EnjiD9+C127rr3q/VqMm+RNCHiaEvBDzdXuVh40D2EwpvQbAxwB8hRDSBSBu+aso/Sil91JK91BK9wwODtY6VYllxuwilH8jQ76iEJUn4Nk5pu3iov40XuuPFvip35EKBBMcmX1gOS4Ojs7i2s0e0XT6fjk7v4yh4hPvuBTnsyYeOeQNMjs9UwCl4SFdA51enlxU/oBH2szv11WChZLNSWh8LlD+HYbg+eue5392rojrPvUQXh6f53/PX7vhImzpD0gqDl2c/L2/zbquBLJFG+fni1jXVT0+eNnGLuw/OYOFCgVfEUM9SZydK/ItH+u1fTqTWmjBH48UfJ8/PYeBjgRf6BjYwnV+3uQz+idzJgY6a9s+k7kSJnNef0S0PsAWpq0DGW77iA10vWmDN35x26dCYfv6rX3IlxxkizaGulMY7g2U/0pETfKnlL7V33A9+vVglceYlNIp//v9AI4CuASe0h8R7joCoPkdNCRWBBi5RnesimIub+GUP+mwkfG+X376ZMh/Fz1nIJi0uKU/g60D3mRHNmcGCBYnpjZfOjMP03Zx3UVe409nUudNXgfH5nD5cDduvmQQSV3h0xdH/auNzULihuXJ2e/NSDYrkH9v2kDRcjHuk/Lp2QKKlou0oYZsH1bwfeVcFlMLJRw+m+Xkf+lQJx79gzeHRgNHwewarvy7ksiVbJyt0GUq4saLB/DimXmUHLcu28dyKCYXTGSL1a8U0oYKXfV032Bnoqrnv2Da6E3rZTbJoJ/A+dwjr+LK/++HGJ3OYzZvVVX+7GpmImt6VwkxVz4sibR1oAM7N3Tiqk09/P0AeAsBs30KlgNDU3gROgo2ywhAyPapVOxdbiyJ7UMIGSSEqP732+AVdo9RSscBZAkhN/gpnw8AqLiISKweOC7lnnp0dHEUL5zxVP/O9Z2wauwQxXA+W8QnHngBX98XuIbML2aDxVjM86L+NAgheO3WvhD5zxWCnZIAL9YHgH/Yu5Ia5osWSraLl8/M46pNPdBVBVeN9PDI3qlpb4ERlX9/JoG5gsV/f1ZUzPnNVkCgflmPwHHfjsoYmmD7+J6/5fIrhLmCxW2fjirWCgPL2J/3f8cNXUlQ6l2xVLN9AG/WDAMbqVAJbG7Pmdki5osWupKVz83rPPZ+/8GOBLd9HJfinP97sosB26W8mUwEO/cfvHgOOdPGd573popWI3/vdoPbPnGNU0z5bxvIIKmrePAjN+KmHYHL0Js2eMHXtNzQyIvyc0xim981PNSd5AKBLQIrDc1GPd9DCBkD8DoA3yGE/MC/6WYAzxFCDgL4BoA7KaXsU/hhAF8EcATeFcH3mjkHiZWB+YLFi5K1cv6MQHdu6Kx7yBfz9UengwayM1HlP7UAQ1Ow0Z/rfv3WPpyeLWBsxuu8ZFv2MfLff3IGG7uTfA685/nbeOVcFiXH5QW4ay/q5bPqx6bzMDQlRKTMehidLiChKehJGfy8OPlnPJJh++GyzH86ofJYpVfw9T6SfN58wQpGLNdQ4+J9WKKJXYW4FDWV/1Wbejjp17J9NvsF75NTCzVtHyDw1gc6E/xqb1Io4LKuX7cC+Q92JqAqBJcOdaEjoeGHL571nq8m+ScwPlfERM6MVf5sQdg+GH811ZfR+XuvUHIq+v0MTP2ztM/fffB6vPfakaqPWS401eRFKX0AwAMxx78J4JsVHrMPwOXNvK7EysOMMJslOrc+CrY49KaNugu+nPzFyYm+8mdWzcnJPDb3pfllOfsgPnN8Gm+/bAMsxyOWc/MmKKX42ckZXCtc4nclNRwqWnzj76tGvFrAdZt7cY8/q/7UtNfOL176MwI6NZ33Yn0+EWb9qwj2uwLeSARCwBdKMVKZSahcWbJFbq5g8cWtFsGK9zk3b0JXSchvrzUvRlcVXL+1Dz86PFFzoWHk/9L4PCitvTD1pg1ofgGVKX+xE5aVfhwaT/5JXcXff+h67NrQhY/efwBP+D0ctSZfvnZbH/73j44CAG9OE/ELV23Exp5UyMaLnve0P/OoaDsVewoY3nPNME5O5XmxN+41VwrkeAeJlkCciV6ryYsVRrtTOiitb747s1ROTYvkH0xRZD9vFC6xd67vRFdSw9PHpnk9Ykt/GgXLweFzWZyZK+LazQH5s4Lvc6Nz6EnrPNHDFognjkzi4OgsLuoLEwUj/9HpPB/M5f0dbJ72YUVESsMqM5Tz9wu+QNB8Npu3uOdfj/Jnrz2ZM5HS1dCCUUv5A8GkyVppn5ShYkNXkqe2oiMdouhO6+hJG9BVhS/4Z4U9AVw3sILiyB8AXr99AH0ZA9ds7uWLZy3l/x9v3YnfusnbRD4udZPU1VDHbxS9GQMl20W+5KBoOVVtHwB47bZ+fPWOG0Ibt6xUyPEOEi0BK6ZmDLVmk9dCyUZCU7iKshwXquJ9X/Qz7amI58ye/8xs0cvOU8rb+5kynslb2CYQq6IQXDnSg0PnsjzmuXNDJ45OLOCBA15vwA1CvK4r5UU9D455Haas6NiXMbBtIIPP/+gIbJfif/7yVaFzY9bB1EIJO9Z1cMtETPv0CQr8yuFuvq9rJqFxsmMFXyCYNCp6/rWsGPYcgLfIZBJaiJRref4A8I4rhvCvL58P7ShVCRf1p3mzXq2rkrdftgFbBzJwXcoXe6b8BzoMwf6hUGs0/YsF2UpD3RgUheATP78bt1y6ns/vbwR9QqNXwXKRqPaqA/QAACAASURBVEH+qwkrf3mSWBVgyn+kN10z6pn3o4QsAcKUIKUUv/GlZ3DnP+wve8yc0Hk7PlfkKZHedJDNn1kolXWTjvSmcHqmwLt72TCzBw+cQV/G4BtzAB6BudTb6YpZPgzXXtQL26X4zddv4VMtGcS4YTrhjVtO6oo/XdO7yhGz7leMBFHNaIcvWxDP+EPT5v2Z8AlNqUtNaqrC1WnKCCv/dXUo/409KXz1jhtqKmrAS1Wx/5daVwq/dN0I/ugdl0JTFV7nOTtXhKEpGOhIwKEB+VdK0zBcvakHhHh5+1qFaYYbtvXXvDqJA/t/m81bvvJfO5S5dn4TibbhMw+9ghfPzIWOMWU+3Juq2eS1ULL9+J/39mNZ/71Hp/D08Wk+c17EnLC/7uh0npPjJes7UbRc5Es2sn5MUMRwTwqTOZOPA97pk//Z+SJet70/RDSMHCgFrhwJZ+nfe+0IbrtsAz5+286yc0sbgV2T9v9lxWOu/IXs+MXrOoL7G8Hs9sHOBCd/poTnCt7GIPX4/cHvESwmwfdqXbZRI9jiJ1uAoL+gFnSV8M1czs0Xsc4v5DLbx6W1lX93Sscl6zox0JGo2jnbCvDO53wJRau257+aIMlfoiFYjou/+NdX8Y8/DY88mM1bUP2ZL7XSPnnTQcbQoPnKn5HB5x45AsBLqpQiEdBZoaZwajqPs/OeX7zTV+5sL9XoDles0eZlf9PyHesDpX/j9rCCFwn2yojyf932fnzh169DOmbmDRCof7bvbWdC8zz/SMEX8CKBQ/6OTWlDxbbBDnz3d2/CTTsGyjxlZvs0QtzixutsQavH728UW4Qiab2qWlUIXOp5/KzPQVUIV/62S/n7oho+dNNW/JvrNy/uxBsAH+62UKrL819NkOQv0RCYRfPKubA6n8mX0JPS0ZnUaqd9SjbSiUD5lxwXB07NYO+xKVw61AVKw/N3AK+BbFNfCqpCMDoTKH9G5myaZ3Q4FstYv3jGI/+NPUlO8jdeHG6nZ1n1dZ2Jmi35UTCbJOUvDt50Taus4At4Cp91sDIff/fGLhBCQuRiqArfCrCWrSKiMxGo/YyhgpCl2Qz8ov5A+dd7ZcL+z22XouS4XtMUIfxKx3UplDrU/K/s2YSPvPniRZx1Y2Ce/2TOREEqf4kLGcyieeVcLnR8Nm+hO60jY2h8oFklsKmROlf+lJPzh97gJTPErRW95y+hL5PAxp4kRqcLGJ8roCet8yImSwFF58gw5f/imTnoqkeu6/3uy82R1A4jsKjqrweM/JkH3Zn0pmsGTV7eebG444auYOKjiJQRfCS3DmRQsBxML5QaUv5Mhaf9rRg7EtqSKH9xX4F6z0/zbTbbdVGyXeiq4l8N+J5/hajncqEnrSOhKTg3X0TRcmvm/FcTZNpHoiEw5T+9UApNVZzJe8VWZnsslBx0p+I/KPmSg4GOhOD5u/x52eiC08Km6oBX+OxJG8gYKk5N55HSVQx1pzhhM/KPKv8NXUmoCsGMPwqAEIIP3rgVukrK/GJGmlG/vx4Eyj9o2JrK5Tn5p3QVSd1rAFMUgovXdaDXJxYR4oYo29dlcPhcFqdnC2UF6GpgRMzO5cNv2o7Lq8wDWiwyCa9eMZkzY0dAx0ET6jwlx4WhKqA0GPZmOyuL/AkhGOpO4sxccc15/pL8JRqCOJHxlXNZgfwtDPckuY2RL9kVd+liA800RSACnySZj8w6XBlmCxa2DGSQ0lV8bd8oXAp89JYd3Ko56Y9N6IukfTRVwYauJE7PFvhVwb99bbxXvKU/jV+7YTPec03FQbMVMeg3G6U5+YcLvoamICMUdz/4hi1473XDZQuQGHHdNuAthBNZc5EFX++5/sObls4e2dKfRtFyaiZ0GLjydzzlzwa9OQ0UfNuNoe4UxmcLa4781841jERbIHbkviKkcmbzJfSkDU5+1Yq+bJSxoQVRT0aSmYSG9V2JMuU/m7fQk9KxqS8Nl3oz0H/3lh2c6Ea57VPe8cl8/54aW0ZqqoL/9u4r+Kz+RsDy5qwg3Mk8f//3Smgq0gmVk39CU/l2gSJEz3/7usY9dUAs+C69trtiuCc056gWeJHf9RZ8w7d9nDqavJYLQz1JjM0UYDl0TRV8pfKXaAgh8vcblRZM27d9dG45VBvuFlX+tuui5LggxFOGwz0p3uEKeEXA+aK3H+/PXb4BE1kTH79tJ1SFcKtmdCaPjoQWm4Uf7k0BJ8o3FWkl2DaIadHzF3L+hqbgd968o+aER6YsVYVgc19A/ovx/OvNwDeDj9+2s6Hd2HQlKPhafsG35Lh8vINLUfdVRLuwsTvFx4CvJc9/7fwmEi3B6HQeH/jSMxUbtcT5+6+czeJ/fP8QrvnUQyhaLkZ609z2qaT8HSHix9M+dqACCSEY7k2HbJ9s0QalQHfawLbBDvzXd10WUtjsvNjwtCiY8u9OLx35sxkzzPvuSHgNYywCqyoEv/KaTTVnvagKgaEq6M8YoeL1YtI+0S7ppUBSV6tO9IxCjPeytI8m5Pxt1+XW0EoBi+UCWFPKX5K/RAgHRmfx+CsTODaRi72dKf91nQnsPzWDzz96FG/bvR5f+Xevxa/fcBEnv4UKjV5s5n2H0OFruy5M2+WqfbgnhfG5AicE1p0bZ9uwblqg3O9nYIkfNm1zKbB1IANDU/iAMKa+pxZKMNTGPmZJ3et6Fa9Uqu0BGwVbECv1JCwnmKVjORSWkPYJop6oK+rZTrApsQDW1HiHlffukFhWMI+6Upcu24jjso1d+NHhCbxt93r8xfuv4R9qnvapoPzZ86YNLZT2KTkuT74M93obhZzPmtjQneQNXpVsm46EjqJlxvr9gOD5L6HyX9eVxMFPvi1I+/gEPL1gItGgVZAyVAx0hsm/EeUvbgu50hDk/N1Qzt+loue/nGdYDlH5r6WCryR/iRAY+RcqkD9T/u++ZhhXjPTgzjduCxXouO1TodGLLQqZhMotAJb2YQp5xCfr07N5bOhO8tEOlci7K6lhMmeWxTwZuPJfQvIHwjYLU+pTucaV/5Z+b0tBXVWQNlTkS05dG7nw1/avOlaiRRGkfah3tRdR/t5mLiuL/YcE5b8S/6aLhSR/iRAYuVdS/uz29V1J3H51eSSylucvKn9DVP6i7eOT9dhMAdddFGwPWYm8mc0RHerGsG0gg//8zt14xxVDsbcvBYLRyqWGi4T333ED/747pfvkX/9HdWu/F4ndKszeWSkQO3xZwVcRxju4dOUp/66khoyhYqGOzVxWE5rdyevThJBDhJDnCCEPEEJ6hNvuIoQcIYQcJoS8XTh+HSHkef+2z5Klnswk0RAC26d6wVev8Allg80qzfTnyt9QecOPzZS/T/6sG5VvZegPjeuqYPswpdtXoeBLCMGH3rC1rkmVrQKzXmbypYZnuxMSNKAx66eRqOfm/jRe/tRtoTlGKwVqJOdvqApUQhoa6dxuEEIw5F+NriXl3+wy9hCAyymlVwJ4BcBdAEAI2Q3g/QAuA3AbgM+zPX0B3APgDnj7+u7wb5dYIWCxvYJVwfP3b9crDN9SFOJZFbWUv1DwLQnJDyD4gLHuWGb7VPL8GTFW8vyXA0ypOy5t2PYRwRa8Vk/kXC4wq8+0XbgUMQXf2iOdlwNsFtNa8vybIn9K6Q8ppexT/hQAtlnl7QDup5SalNLj8PbrvZ4QMgSgi1K6l1JKAdwH4N3NnINEa1Gr4Gtx8q/81skkKg93ywnKX48qf/9nb/QCYPoL0GzeQtpQQ6MPRDDyr+T5LwfEKZfNJETYgtdIwXclg/2fs/cXL/gKnv9Ki3oCkvxr4YMINmMfBiDO/B3zjw3730ePx4IQcgchZB8hZN/ExEQLT1WiEko1PX9m+1T+gGYMtWKTF7OT2KYn3nMGQ74A7zI7oSko+gvRbMGq2qDFiLaS578cEJV6ognl373WlL9P7Ox9wHL+jjDYbWUqf8/2uaA8f0LIw4SQF2K+bhfu8wkANoAvs0MxT0WrHI8FpfReSukeSumewcGVuxHyYnDf3hP41S8+tdynUYYg7VPJ869T+fsK/6cnpvG6//6vmMx5/j1bFDzPPxjvYAq2D+ApLFH5Vyd/v+BbwfNfDqi+/QWgqf1c+zMGEppSNgButaJM+avEK/j6TcLuCvT8AW8nsB3rOtpaN1pq1JQTlNK3VrudEPIbAN4J4BbfygE8Rb9JuNsIgDP+8ZGY4xccXjozz/c/Xc5z+MCXnsZ3P3oTnzNTM+fvK3+tTtvn0Pg8xueK+PGrE3jPNSOB8jc0nu1mUU+R4BKawj3/bNGq2kXK7B42YmGloDOpIV9ymiL//+fGrXjDjoEl37GqXWAFXyYODE2BqoC/F1aq7fO67f146GNvXO7TaCmaTfvcBuAPAbyLUioOYP8WgPcTQhKEkK3wCrvPUErHAWQJITf4KZ8PAHiwmXNYrShaDi9yLReOTeYwmSvxiZhAoOwr5fxLNQq+QNj2YamfnxyZAuB1/uoq4Zf7AEt+hEkyoako+sq/aLtIVhlV8J5rhvE3v7FnSTYsaQbMqmlGtW/oTuKmHWvnqpe9b9j7SxfSPsz3X4m2z1pEs0biXwJIAHjIVyZPUUrvpJS+SAj5GoCX4NlBH6GUMjb5MIC/BZCCVyP4XtmzXgAwbXfZyZ8Rfa4YWDy1lb9P/lUacTqSOk74CwpTeE8emQSlFHnT5mMHVMUr7LIOXzEVk9QD5W9aDpJViL0zqeOWS9dX/2WXAR3+1Uozyn+tgQ3zy1tCwdef7cN8/5Vo+6xFNEX+lNKKg8IppXcDuDvm+D4AlzfzumsBK4P8vdefLwb745qs4Fsp6umfs16F0Lxxxh7ps3TPmbkiTk7lsSA0LBFCoCsKLDec8wfCyn+1bp/HNjVvJuq51qBFlD/P+VPKPw9S+bcH8l25TChaDlc6ywWm/LMxyr9SwZfZPtV8WW/zcm9ByZk2v9T/ydFJ5Et2aOtCTSWwbDeG/APl722isfreqmyRk8o/AFP+7IpQ17ycv+0G5L8SPf+1CPmuXCYw5U+XcQGw7HLyjxvvMJsv4X/98DAcl/KCb7W0T0fC28fXclwsmDa29GewoSuJJ49MYcF0kBZii7qqCBt7BItCUlcF8ndXZWclSyFJ8g/AlD97fyXUGNtHkn9bIN+VywS2ycdyOj/MwskKtg9X/oLt88ih8/jcI0fw6vksLMeFQqp/QFlDUq5oI2fa6EhqeP3F/Xjy6CRyph3aZERXSVmHL+Apf17wXaW2DxvGVqk57UIEqxWJOX9m+/CCr/T82wJJ/suEouWRrO3WvwtSq1GqavsE5M9uNy0XlkOrxjyBoOkqZ/rkn9Bw4/YBzOQtvHhmLjRnXlcVlGzveUPk7xd8Xdeb/rga56h3SOVfhqjyZ+MdKA3EiFYlSSbROsh35TKBK//l435u4YjKP872YbeXHM/KqVXAZF53tmhjwbSRMTzlD3iLnjhnXlMJX2jESGRSU2HaDrd+VqPnzwq+a6VBqxVQlTD5G77nDwTvPan82wP5rlwmmCtA+ccVfM045W8Gyt923JrKjHndOdNGrujZPkPdKWzzRwxHlT9LBImLSkJXULRcbv2sRs+fF3xl2ocj6PAVm7x88rel599OyHflMoGR2nIq/1jbxz9Wclye6ee2j+2g5NCqxV5AVP4Wt30AcPUf8vwVJUQEDAnNG+9Q9K+QVqXnz5T/KrxqWSqw3o68EPVkSr/keMdkzr89kO/KZQJT2Mup/O2YnD/z/IEg6x+Qv7cg6DWUGSO9bNHGQsnhNs+N2wcAIJz20QjvAo7z/FltZDXaPqz2IZV/GN6CL9o+3vGSVP5thXxXLgMopZz8m8n6j83kK+6YVQ/ibB92DAisnxzz/G3P86/W4AWIu1iZcFzKUy+v296PlK5iQ1ewJ6omKn81ovxtl9+WXIWJGZnzj4dY59EF5c/ee5L82wP5rlwGmIK6bkb4/+Lnn8S9jx9b9OOtmIJvyXY5eTN1Jto+Vh2Dtzp9sj87VwQAdPjKvydt4NE/eBN+eU8w289QFT4HKBr1BID5gk/+VWb7rFQMdSehKoTvTCbhQVUItxdFz58dkx2+7cHaGBK+yiCS/2JtH9elOJ81MeNvcbgY8Nk+pg1KKQghKNku+joMZIs2V91ltk8NGyOpex/o8XmP/DOCzRMlQk0lsZ4/8/jZLl6rUflv7Elh711vweAaGgPcCojvH7aBOxBYjrLDtz2Qyn8ZYAoNVItV/syPFz36RsHI36XetE3AU19sdj63fcxwzr8W+RNC0JHQBOVfWWPoatj/ZQiUv0/+q9DzB4B1nck1M465VRDJXVdJufKXf6+2YHV+olY5WqH8mddfchZP/qzgC3jWD6UUlkPRk/Lm4zNSZgVh03Zg1RH1BDzfvz7yD54roZaT/2zBu7JZjWkfiXgw8WCoCgghPN3Dxo1Iz789kOS/DGANXkCwiUWjYGrcchZfMBYXjmzR5j93pz3lny85cF3KX4sXfOtIr3QkNJzPlts+UYQsgBjbZzZvhX6WWP1g5M7+v5WI8pfhqPZA/pmXASy+CAQt7Y2CK39hIWkUdoj8LW4h9TDbx7KxULLB1ifP86dVN3Jh6ExqfGGqtvm4VoH8mfJnnv9qbPKSiAe7cmTvI7Us7SNpqR1odievTxNCDhFCniOEPEAI6fGPbyGEFAghz/pfXxAecx0h5HlCyBFCyGfJBWiIisp/sTP9WUKmGeVvORTsCnu+aPPnYp5/vuRw1Q945N+I8o/7PgpxIdFDHb6+8l/lnr9EOdhwN7bYl3X4XniUsCxo9hP1EIDLKaVXAngFwF3CbUcppVf7X3cKx+8BcAe8rR13ALityXNYdTAF5b948me2T3MFX7b/bbZoB8o/HRR8w6MfHG+wWx3KrFPYc7eq7aNUsH2Y8pe2z5oDU/6VbB8p/NuDpv7MlNIfUkoZOzyF8ObsZSCEDAHoopTu9Td7vw/Au5s5h9WIYiuUfymIXy4WluOiN83IP7B9ROUv9gAw5W9otZUZs3oIAdJViFsXnsuIVf5ewVcOR1s7YGkfdqXHlH4Q9ZT/1+1AK//KH0R4P96thJADhJDHCCE3+ceGAYwJ9xnzj8WCEHIHIWQfIWTfxMREC091edEK5Z9rUPkXLQf/9LOx0OYxlkPDyt+frZIyNBh+BDM69M1261T+vtrPGFrVph3xuRIVPP+krsi45BqCJqR9gKDAKwu+7UXNPzMh5GFCyAsxX7cL9/kEvI3av+wfGgewmVJ6DYCPAfgKIaQLQNwnuCL7UUrvpZTuoZTuGRwcbOT3WtEQ1Xq7bJ9/eOokPva1g3hpfJ4fsxwXnUkdqkJ85e+di6EqSBkqCiWbk7+hKjAtb7vFRjx/cXxzHESrJ7bJK29Jy2eNQYumfSJRT5nzbw9qdvhSSt9a7XZCyG8AeCeAW3wrB5RSE4Dpf7+fEHIUwCXwlL5oDY0AOLO4U1+9KFqtK/jW0+RFKcXX9435rx3c33JcJDQFHQktFPU0NIK0oYaUf3+HAdN2YLtu3WkfoHqxFwg3/MQ2eRXt0CwgidUPPaL8WQ2gJGf7tBXNpn1uA/CHAN5FKc0LxwcJIar//TZ4hd1jlNJxAFlCyA1+yucDAB5s5hxWI0LKf5E5/4UGcv4vnJ7H4XNZAOF4p+1SaCpBZ1ILFXwNVUXKUJG3HL4Re3+HwXfcqqfJq8Mv+NYi/2irP4Oo9mXSZ22hLOdPJPkvB5r9VP0lgE4AD0UinTcDeI4QchDANwDcSSmd9m/7MIAvAjgC4CjCdYILAi2JepaCxisR337uDE5MLoSOfWP/KP9e7CuwfAunM6mHCr6GpiBtqDztQwjQmzYWFfWslvQBgqinQsKZ/0SMBSSxNqCrkYKv3MxlWdDUYDdK6cUVjn8TwDcr3LYPwOXNvO5qR7ElBd9gFo+I3//6QfzitSP4k/dcwY995/lxjPSmMDZTCN2fbczSmdT8nL9A/rqGvO/5dyQ0JHUVk7mS3+RVT9SzPtuHWwCRNI8k/7ULLZrzjzZ5Sc+/LZDX08uAVij/fEzBl1KKouXi+ERY+U8vlLB9sANAeJ4P8++7fNuH2VG6SvyCr6f8u5I6DE3hs31a6vlH/F/xOKsHSNtnbaFizp8VfKXybwvkp2oZ0NKopy0qee/744Lt47gULg1SN6Lnz2yfrqSO+YLFH8+KwDN5C9mihc6khoTmpX3qjXrWa/sYnAjK1T1T/1L5ry1EC77RDdzlSOf2QJL/MiDU5LXYgm+pfKonU05n54t8Rj77QKV0rez+ll/w7Up55G8JBd9rL+rFqek8XhqfR0dCQ0JT+WvWszMVa/KqNtcHCJR/XBMXa/SSc33WFnjBV40v+MqoZ3sgyX8Z0JrxDsFsH9a4JRZ/T0zm/du9Y2mDKX/vvt74ZheGqqA7pSNr2ij4EVRdI3jb7vUAgLGZAlf+LGFUjzLrSuowVAX9fhNZJVTy/IFgxINU/msLvODrd3dHN3ORBd/2QJL/MqBou3ygWrNNXkAQ9xQjpMz6Ybcx8meLgeNSUOqRLxvnMJXzRikYqoJNfWns2tAJwJvTk9AU/lz1FHyTuop/+g+vx7+5fnPV+zEiiNvknCl/6fmvLfCCr6r6P4eVv7R92gP5qVoGmJaDjOHZIc2OdAaCD00pRP457/mZ7cPI33899rqaSjj5T+S8+ftMhd/qq3+m/BnqKfgCwOXD3XVEPSsrf/aaiVW4haNEZZQVfCNpH1nwbQ8k+S8DTNvlZOwugvxdl2Kh5HDSZl696Ocf85V/qcz2Cd/XEJT/RNb0jkXIvyOpcRUOhPP4zSLa6i+Ce/6rcPN2icrg/+dq2PaRI53bC0n+y4Ci5XBFvBjlz/bv7fVHLzPFJNYSTkRsn5R/pcHuy7x/TSF8565J3/ZhY5avGO7G+1+zCbfsWh+yZeqxfeqFrsVHPQEh7SOV/5qCFrnaY//1plT+bUVTTV4Si4Npu1yJL2YbR2b5dKcNYCrPvX42lXO4J8U9f6b02VhlthiwRUDXwspfVwn/8BFC8KfvvRIAcOhsMBCuXtunHkQ39hARRD2lRllL0GsMdpOef3sgP1XLAJH87UXsxMXIvy+q/P0Pz64NnZjJW5hZKJXZPlakPqBHbJ84BQ4g4vm3UPmrlW2fJC/4SuW/lsCUf9l4Bznbp62Q5L8MMC0Had+GWZzyZ7aPF6Nkap4R+sXrvW7esZkCv83QFCgkWGyY3aQLBd+C5VTM8ItF11Yqs6gFEH5N75jM+a8tVBrsZsmcf1shyX8ZYNou77hdTNSTdff2+OTPSJ8p/56Ud7xoO9z20VUFuqrAcr2fLeF4Ulf5B7GSqhfJWa+jyateGGo1z9/7GyWk7bOmEI33RnP+0vZpD+Snahlg2oHyX0zBl9k+rOAbjXqyrlrLdvltnPztsOfPMtdM/VdW/gL5t3CbPa1Kzp95/dL2WVsoG+xWtoevJP92QJL/MqBoucg0EfVkYxZ6Msz2CZM/20LRdFxu8+gqgaYS2Fz5MzvI+6DVJv+AgFta8K1q+0jPfy2imvKXfn/7INM+ywDTdnj0cnHKPxz1jNo+bKiaZbtwhc2ydVURop7BFQEgkH+lgq9gvbQy51+t4MteU3r+awuqEin4CrN9ZMa/fZDKv81wXArLoUjpKghpLuoZFHyZ8vcWBWb7lBw35O3rCgmKww3aPuKiUGmBWAyqz/aR4x3WIiqNdLYcKpV/G9HsNo6fIoQ85+/i9UNCyEbhtrsIIUcIIYcJIW8Xjl9HCHnev+2z/naOFwzYLP+E7s2rX4zyZwVf3uEb6dplyp9tuwh4ClvXFK74K9o+dSn/VqZ9qs32kZ7/WkT0ao8RvuNK8m8nmpVUn6aUXkkpvRrAtwF8EgAIIbsBvB/AZQBuA/B5tqcvgHsA3AFvX98d/u0XDFgXblJToBCyKM8/X7KR0lVOirzJi3n+yaCbV1T+mqD87YYLvkvj+Rt1RD1lh+/aglbB9gEAyf3tQ1PkTymdF37MAGBMdjuA+ymlJqX0OLz9eq8nhAwB6KKU7qXeHOL7ALy7mXNYbWBEndDVJpS/g0xC5cQpTvUkJJiFU7LdUCev6PlbEc+/yyf/SlHPpWryShsahrqT2D6YKbtt+2AH+jMG+juqj4WWWF0wIp3bYnislfUkiepouuBLCLkbwAcAzAF4s394GMBTwt3G/GOW/330eKXnvgPeVQI2b64+Gni1oOjP5UloChSFLCrnX7JdJLQgmy+mfQxVQcIflVtyKAjxbR+FRMi/sbSPeLyVH1BDU7D3rltib7v5kkHs/8+3tuy1JFYGbt4xiE++czcu3dAFIKr8pfRvF2p+igkhDxNCXoj5uh0AKKWfoJRuAvBlAL/NHhbzVLTK8VhQSu+llO6hlO4ZHBys/dusAjDln9RVqIskf8d1oamE2y9i2sfQFE7UIeWvKn7UswU5/xbaPhIXHlKGig++YSsv9Io+vxT+7UPNPzWl9K2U0stjvh6M3PUrAN7rfz8GYJNw2wiAM/7xkZjjaxLjcwXc9D8ewampPD/GC76a58EvZhtH2y+M6RHlb9ouEprCydkScv7eYhFj+2hh8k9UtH0Ez7+FTV4SEoQQ7vXLqGf70GzaZ4fw47sAHPK//xaA9xNCEoSQrfAKu89QSscBZAkhN/gpnw8AiC4iawbHJxYwOl3AMX9jFcBr8ALYrB0CZxGD3WyHQlMI9/zFDt+EpkJTvTk+pbIO36Dgy1NAStj2qeTn6yoB+1y2cryDhAQQqH9VXlW2Dc16/n9KCNkJwAVwEsCdAEApfZEQ8jUALwGwAXyEUsp2Lf8wgL8FkALwPf9rTYJZPOLkTmbRJDS1KeWvKQonavacJcflto2hKSg5LjQnaPLSFAW2E97YvazJqwKxE+ItNqbtytkrEi2H5/VTqfzbiKbIn1L63iq33Q3g7pjj+wBcLxru9AAAH9tJREFU3szrrhaw4q6Y6GGka/gF38VEPZnnryrel9jkxa4GdFXhBWCFeMrKs31Y1NNX/lp95A94VpVpuy1N+0hIAIHyl3N92gf5KV5CcOXvumXHdJUsOuppC80whkDopu3yxqiEr/wtJyBrz/YJN4Rpddo+gBdPZQuJhEQrwRS/vKpsHyT5LyFYcdeJUf486rkY28ehvOiqqySwfXyl7x1X+FRP8Vg07cPIPqkreMuudbjuot6Kr5vQFKn6JZYETPHLqGf7IAe7tRiuS+FQCt33x4F4z5913C6m4Cu2wTNvnz131PO3HcpHKGjCQmE7NKTiCSH40m++purrGpL8JZYIakzsU2JpIT/JLcZf//gYfu4vfgxA9PwD2yfk+ZPFKX/L9/wB3/axw1FPADzWGbJ9FEUY6dy4d+8lieSHU6L1YIpfkn/7IMm/xXjlXI7n+tkcH9HXL0WarhbX5EW5N6pXUv5+wddyaED+GuFXIeLxeiFtH4mlAntbSfJvH+QnucWYzXubpjsurWr7GJoClSyO/G2H8pnohtC45UU9Vf78ps2UPyumBQuFeLxeJDSF9wVISLQSrNNcRj3bB0n+LcZMvgTAs3ziop6MfA1VWfR4B9sNsvZepDPYwD2hhRcF0d4xNEVQ/o3bPoamyAYviSUBaxqXUc/2QX6SW4zZvAUAKFiOoPwFz99m8/WbIf+giCvaPqbthAu+vu3DBrFpQk/A4mwfVUbxJJYEMurZfsi0T4shKn8W9Qwrf4c3Zy1+sFvg+SciBV8jlOmnsBwXBk/7eFFPSumibJ9fuGoI5+aLDZ+vhEQtxA15k1haSPJvIVyXYq7gKf9iSPmHPX9x42pLuCqoF6Lnr2uEF5ZDtg9X/oLt45O97VLPOmpQ+d9+dcXp2xISTYEpf5nzbx+k7dNCzBctMCFfKLnc83dCUU/KFbdCFtvhG6h2XfVsH0ppKOppaCqf6qkJyt87Bxclu3HbR0JiqSBz/u2H/PS3EDO+3w8ARTtQ/pZA8N7MfS+Roy1yto/thMc7iHv1Glpg+5h+h28w3iHY+ct2AztIQmK5IXP+7Yck/xZi1vf7AaBQcrgdEx3vwEi3qYKvkPO3nGB0MyN/NtvHI/lgQWDnYDmN2z4SEksFrvyl7dM2yE9/CzErKn+h4Cv6+mIjVlMFXzWIdJYcNzQqGhA6fG3B9vHrBLZDYdm04YKvhMRSQdo+7Yck/xZiRlT+lsM3bilT/iL5L2a8gxPk/L3xDpQvNOUdvuGpnuzxlitHM0usHEjybz+a3cnrU4SQ5wghzxJCfkgI2egf30IIKfjHnyWEfEF4zHWEkOcJIUcIIZ/1d/RaExA9f9NyBeUfTvvoPO2jLFr5q9z2IX4BN2ge844rnOSNMs/fXVSTl4TEUkGVnn/b0eyn/9OU0isppVcD+DaATwq3HaWUXu1/3SkcvwfAHfC2dtwB4LYmz2HFYDai/FnBV0z7iLttqQQNkz+l1G/yCm/awm0fXezwpTCtYAgcI3vbpd5YaGn7SKwQ8A7ftaMFVzyaIn9K6bzwYwZAVSYjhAwB6KKU7qWUUgD3AXh3M+ewkjCTL/GoZVGwfaI5/2aUP7s/t338wq4ZUf5sgSmUHP56bBFge/vKgq/ESkFg+yzziVxAaPpPTQi5mxAyCuBXEVb+WwkhBwghjxFCbvKPDQMYE+4z5h+r9Nx3EEL2EUL2TUxMNHuqS46ZvIWh7iQApvzjt3FMcM+/ceXPniu8k5dA/oLnDwALJbvM87dd1vkrP2kSKwNB1FO+J9uFmn9pQsjDhJAXYr5uBwBK6ScopZsAfBnAb/sPGwewmVJ6DYCPAfgKIaQLQNw1XUX2o5TeSyndQyndMzg42Ojv1nbM5kvoyxgwNAVFyxVsn/BgN1H5N9rkxZ5LbPJyabB3QEKY6gkALg3fF/BmDdkOlXNUJFYMpPJvP2qOd6CUvrXO5/oKgO8A+C+UUhOA6T9+PyHkKIBL4Cn9EeExIwDONHTGKxizeQsbupJIagoKJZv78GLU07KpMN4BcBtM+zALiY909kk+W7RDP4vFXG77+I/he/vKCZ0SKwS84Cs9/7ah2bTPDuHHdwE45B8fJISo/vfb4BV2j1FKxwFkCSE3+CmfDwB4sJlzWEmYzVvoSRtIGSqf8QPEKH8tIGNx4mc9YDtxiSOdASBneuQvzvZh0KK2j0NhOVTaPhIrBsFgN/mebBeaHez2p4SQnQBcACcBsFTPzQD+mBBiA3AA3EkpnfZv+zCAvwWQAvA9/2tNYCZfQm9aR1JXMSuQvzjeQRzsphCCRpOevODLt3H0/l0ww8pfJH+jLO3jhnoFJCSWG5q0fdqOpsifUvreCse/CeCbFW7bB+DyZl53JcK0HeRLDnrSOlJ6VPlHo57Bhuri/r71wIpJ+wAxyl+IcZanfagc7yCxosCUv9zMpX2Qn/4WgY126EkbSOgq5oSGL7HJS0zZKISgQe6HE/H8GbFHPf8424cngEwblkPRmZQTvSVWBqTn335I8m8R2GiH3rSBlK6EbB/HrZTzR8PjHdiVQjTBw22fSDevdyw80nlqwQQAdKX0hl5bQmKpwNI+0opsHyT5twhM6XenPM9ftH1C2ziGZvt4TV60gQWgLOcftX10P+oZm/bxHjOVK/FzlZBYCWA5f2n7tA+S/FsERr6dSQ0pXeVq31CDLL/r0tDeuewSt5GiL4t6stgm8/gncyZ/PQChGCe3ffh9JflLrCwwrSJtn/ZBkn+LIJJ/0lffAJBJqJywLTfchavxjtv6jf/oeIdrNvVCVwmeOjYFILCDwso/fEnNbR/p+UusEPAmLzlvqm2Q5N8izPsF144y8tc4uUcnb7JL3UaKvmwBYR+S7rSOm3YMwnIoEpoCNiQ1oZXbPnrkKkEqf4mVAkUWfNsOSf4tQs4n/86EF/VkyBgat32iWy0yJb4Y5a8LzTA/f8VQ6HmB+A5f9hjp+UusNMh5/u2HJP8WIWdaUBWCpK4gqQd/VtH2KUWGr7HiViPKPxjvEHxIbr1sPQxVCan98EIQ9BUAAfnLtI/ESgF7P8uRzu2DJP8WIVe00ZnUQAgJK3/B9mEzfqLpm0aUPx/vIHijXUkdb9m1LqTk42f7+E1ejou0ocrNXCRWDJjdo0nPv22QFb8WIVu00ZHw/pyi59+R0LhVEx27zJR/NOs/vVBC2lBDz8NgRwq+DH/63iswX7D5z3EWECEEukpgOVRaPhIrClL5tx9S+rUIWVMgfyOs/JnXz5S/EUnfRGf6v+fzP8HnHz0a+zpOJOrJ0JM2sLk/zX9OxNg+3vfecUn+EisJivT82w5J/i0Cs30AICkQr6j8o54/u9QVyd9xKU5N5zGRLca+DrN9an1I4mwfIFhwupKS/CVWDuQevu2HJP8ayJdsfPT+A5jImlXvlzNtdPqEmvKVv6oQJPyN1IFyz1+NKfjOFyxQ6m0AHwc7splLJagKCTZ5j1kIZLFXYiWBK39p+7QNkvxr4OXxeTz47BnsPzlT9X7ZohXYPv5uWglNgaaScuUfIX+x4MtmBJkV5vw7bnnapxKC+T/S9pFY2ZDKv/2Q5F8DC6a3PSLbj7cScqaNDt/2Yco/oSl8q0ZKKUpM+UejnkLBd8afEcQWiiisCp5/HOKGvLE0hSR/iZUE9haVs33aB0n+NZAveaRfiYwZskUbnTzto/j/qtCFom5U+QdRz4D8Z5nyr/B6TkzUsxLitnQ0uO0jg14SKwdsRLmc6tk+tIT8CSG/TwihhJAB4dhdhJAjhJDDhJC3C8evI4Q879/2WUJWtslXsLz4ZCUyBryFwbTdsqhnQlP4GAbbDZQ/j3rGFHwD5R9/pVEp6hkHvsgIC4VU/hIrEVz5r2w6WFNomvwJIZsA3ArglHBsN4D3A7gMwG0APs/29AVwD4A74O3ru8O/fcUisH0qk7841A0QyV/lIxVslwpRz7DKcRpQ/nEdvpUQp/yZXSTJX2IlQZGef9vRCuX/GQAfByCG1W8HcD+l1KSUHgdwBMD1hJAhAF2U0r3UG2J/H4B3t+AclgyFUm3PP8eHuvlpH0b+usLfzI4T2D66Fi74hsm/uufPlX8d3bmM9EMTPv3XllFPiZUEuZlL+9EU+RNC3gXgNKX0YOSmYQCjws9j/rFh//vo8UrPfwchZB8hZN/ExEQzp7poLJQ8Yq/m+WdNj7CZ7cPIP6mpPGljuS5KbLCbWpn8Wdqn0utxz78B5S/aPqwG0Z2W5C+xcsA7fCX5tw01q36EkIcBbIi56RMA/gjA2+IeFnOMVjkeC0rpvQDuBYA9e/Y0tt9hixAo/yq2T7GC7aMrvJAVV/CtpvwrvZ7VgO0TnekjHpO2j8RKghzp3H7UJH9K6VvjjhNCrgCwFcBBv2Y7AuBnhJDr4Sn6TcLdRwCc8Y+PxBxfsWDKv1LTFRBsns7In41WSGgKJ17LcQPPv4rtU1v5syavOqKemgJDDWb8A7LgK7EyESj/ZT6RCwiL/lNTSp+nlK6jlG6hlG6BR+zXUkrPAvgWgPcTQhKEkK3wCrvPUErHAWQJITf4KZ8PAHiw+V9j6ZCv4vk/cGAMv/43T/OCL7N9FL+zN6GrnGxF5c+sIE7+MTn/SjUG5vnXc3XMmsxE8A5f6flLrCDwqZ6S/duGJQl7U0pfJIR8DcBLAGwAH6GUMjb7MIC/BZAC8D3/a8WiUCXnf+DULH786iSu3dwLALzJC/AavbwmL6b8vbSPQoJirVol51+x4Ou40BSCehKyuqqUXSHoKoGhhvcckJBYbgSD3Zb5RC4gtIz8ffUv/nw3gLtj7rcPwOWtet2lxkIVz79oebftOzkNwNvFi+Hazb24bGM3J1+m/MVRy3wD9zjbp8p4h3pnnhsx5K+pCrpSel2Lh4REu8DsUZnzbx9km2cNFJjnH2PDFPw6wIFTs9D8XbwYvvSbrwEA/ODFswA8z7/kuCEyjnr+RctB0XKR0BSYtgvXpWXpB9uldV8a65pSNgDuyuHu2Kq7hMRyQlGk7dNuSPKvgWpNXkz550sOetLxalqPeP7inP0o+bOkz/quJE5N51FyXCSV8IYutuPW3QhzzaaestrAv3/j9roeKyHRTrCrYMn97YMk/xooWJU9f0b+QFDsjULlHb5e2idutj4r+DLLZ31XAqem8zAtt2w3L9ulNcc5M3zwDVvrup+ExHJj54YO7NrQiZGedO07S7QEkvxrYMGsPNunHvJnTVW2U+75KxHlH5B/0ntNxwEQTuU4LpUt8BJrDhev68T3f+/m5T6NCwryIqsGqo13KArZ/0rRSTHRYzk07PmTyrYPEN9bYDn1e/4SEhISlSBZpAoopcj76j6OiAuWg60DGQDhmKcIFuu0XQrTdkNzdqJRT9H2AeITP47r1p32kZCQkKgESf5VYNpusAtXDBEXLQeXbeyCqpCKtk8wudP3/GMKvm5E+a/r9JR/XJ3BlraPhIREC7Dmyf+/ffslfPHHxxb1WGb5APHKv2g56Erp+LXXbsZbdq2LfQ6m0i3f80/EFHy58l8oIaWrfKOVuDqD7VA5+VBCQqJprPmC7xNHJrGpL41/d9O2hh/L5vp4uft4zz+pqfjkL+yu+ByaMNjNclwk9PKCL9vGMVu00ZXSYKhewqeS8peev4SERLNY8yySSWjI+yTeKJjy700bsSq8YDk1xyQEyt9r8jLilL8/qXOhZCOT0PgCEbfgSM9fQkKiFbggyD9nVt98vRLYaIeetF6mwi3HqwekIjn8KMTdukp2OOcfVf4Lpo2MofEFQnr+EhISS4W1T/6Giry5OOXPrhh60wZsl8IWir4s4x9twoqCp30cbw9fMecf9fwXSg4yCZUr/1jydyjfGlJCQkJisVjzLJI2ND6WuVHk/SuG3oyX4RcTP6zzN2nUp/zZHr6i7RPdwD1fCiv/OKtJNnlJSEi0Amue/DMJlRduGwXL+PemDQDhxA/7PqnV8Pw5+btlHb5aJOqZNx2kExq/T5zyt6TnLyEh0QKsefJPGxof0dAoCoLtA8Qr/1RN5R/YPvmSE7KJok1eCyUbGUNFQvPuE1/wlVFPCQmJ5tES8ieE/D4hhBJCBvyftxBCCoSQZ/2vLwj3vY4Q8jwh5Agh5LNkiQfLdyRUnrFvFGyiZ4+/2bmo/Lnnr9Xy/IO0T8600SV0AhNCoBCx4OsgbQTKv1LOX5Wev4SERJNoOudPCNkE4FYApyI3HaWUXh3zkHsA3AHgKQDfBXAblnA3r7Th/Yr5kg1DMxp6bCFq+whKnMVAayl/pu7nChYoBTojM4BUhcB2KSilWCjZ6EiofOxzXFex7bp1T/WUkJCQqIRWSMjPAPg4AFrrjoSQIQBdlNK9lFIK4D4A727BOVREJuGR88Iiir4Lpu2NbkiWd9wW/e9r5vx98md783ZGZgCpCoHrUhQtF5TC8/xZwTemq1hGPSUkJFqBpsifEPIuAKcppQdjbt5KCDlACHmMEHKTf2wY3kbvDGP+sUrPfwchZB8hZN/ExMSizpEr/0X4/vmSg7Shcp8+RP7+VUGihu3DiJrtzRsdAKcST/mzonTGUKEo3j678YPdpOcvISHRPGraPoSQhwFsiLnpE/i/7Z1tbGRlFcd//860w/ZlZbvbxYVl32QxokZcKpIofBEViLwnZokRjEbUaKLxFSQxRD8YNWpiohAIiO+oQYUYP/gSg19428VdWURkgVXXLbC2XejOtp3t9PjhPnfmznSmnel0904755dM5va50+npudPTc//PeZ4DXwTeVePcCLDJzEYlnQf8RtLroWYHwbp3DGZ2B3AHwPDw8IJ3FrWIN1w7uqjgP0NvTyZRelm+e5hqcMJXEtkulXbsrJZ9urpEcdZKk9LxP6uebFftzL9opbUDjuM4i2XB4G9mF9cal/RGYCuwN8zZbgQel3S+mb0ATIfv3y3pWeBsokx/Y+JtNgKHWvoNFqA3BOfF1PpHmX9yu4XmF3lBNOk7nq8t+2S7xMzsbGlyuS9XDv6F4lybZ2ZnPfN3HKdlFp1CmtkTZrbezLaY2RaiwL7DzF6QNCQpAyBpG7AdeM7MRoAJSReEKp/rgftb/zXqEwfTxZR7TgbZJ56ATWbipQnfRoJ/V1c586/a+rkvlyU/XSytJo7nKHLZrooKpd3/GuOlV6Z8kZfjOEvCidrV8yLgy5JmgCLwUTMbC+c+BtwDrCKq8jlhlT7QWuafD7JPrOsnNfhGJ3whyvyPlCZ8K2WfNb09jB8rlCakK2SfRPD/wPcf49odG+d0A3Mcx1kMSxb8Q/YfH98H3FfndbuANyzVz12IVjT/yUKRU3t7Epn/3FLPher8IZJ24n8c1bLPmr4exvOF0p1Jrcx/slBkYmqGwxPTnvk7jrMkrPgUsjdXrvNvlny17FNR6lmkJ9tV2plzPuJVvl0q34nErOntZvzY8XLwr5H5j+anARjLFzhedM3fcZzWWfHBP9bk84vY1nli6jj9uWxZ9kkG/0KxIb0fyuWe/bks1QuaY9knlqXiOYpcNlP6eaNHo/mCsXwhKvX0RV6O47TIig/+mS6xqjvTdOY/O2v872iBoYFcnWqf2Yb0fqC0Irda74co+E9MzfDyZDQnEN8Z9GTK3cPG8lHwH80XwiKvFX/ZHMc5wXREFIl29mwu8x8/FmXZQwO52nX+M8WGyjyhnPlX6/1Q3i760JFJMl0qSUw9Cc1/NB9n/pH847KP4zit0iHBv/mdPQ8fjQLt+oFT6OoS3RlVZP6TTcg+cXVOreB/atg36OD4JH09mZIslEto/nHQD5t/uuzjOE7LdETwj7Z1bi7zPzwRBdyhgRxQqcFDVOqZazrzryX7RGMHx4+V9H6onfnHeObvOE6rdETw7+tpXvOfG/y7KmWfQpFVDWr+2Xky/3jH0ENHpioqgXLZTLna52hl8HfN33GcVumIKNKbyzat+dcM/scrSz0b1fyz82r+5UYx1Zl/WfYpVNT2+5bOjuO0SkcE//5c803cD09Ms6o7Q19cfVO14rYZzT9bKvWsL/sAVZl/F4VwpzGaL7BlbW/pnC/ychynVToi+C+mlePho9MMDeQSE7DVmn8TmX+mfua/qjtT6tzVn8j8c9nyls5j+Wm2rx8ov58Hf8dxWqQjgn9fT/OlnocnpkuSD0Cuu0rzb6LOP17hu7pG8JfEYND94319oHynYWaMHS1w+qmrSncaWdf8HcdpkY6IIr257KImfNcng3+V7DNVaF7zr27kEhP3CI739Yl/nlm0J1G+UGRtfw+DYX7ASz0dx2mVjgj+fT3NN3GPZZ+Yas1/UYu8amj+UK74qc78AV54eQqAwb5y8HfN33GcVumM4N/knv7TM0WOHDvOUH8y8y9r/jPFWY4XbUkWeQGloN5XoflH7z0Sgv/aRPB32cdxnFbpiCgS75SZb1D6ievqh+bIPpHm38xe/jD/Ii9IyD6Jap848x95eRKAtf09rC0Ff8/8HcdpjY4I/r255hq6VNf4Q6Xm30wXL5i/2gcSsk9VtQ+UM//BvlxpTUDGNX/HcVqkpeAv6VZJ/5W0JzwuS5y7WdJ+SU9Lendi/DxJT4Rz31H1HscngFLm36DsUyv4Jxuqx/17G93eYb5FXrBA5n9krubf7bKP4zgtshRR5Ntmdm54/A5A0jnATuD1wCXA9+KevsBtwI1EfX23h/MnlL5SQ5cGM/+jtTL/TKnuPpZ/Gs/859bxJ5lP8z8wmqc7I1afki3JPj7h6zhOq5yoHr5XAvea2TTwvKT9wPmSDgCrzewhAEk/BK7iJPXx/dwv91YE2HrEzdbX9VfKPuPHCrzzWw+W5J9Gq326Q0+BbJ3eu+Vqn7mZ/yPPj7F1XR+SSrKPl3o6jtMqSxH8PyHpemAX8BkzGwfOAB5OvOZgGDsejqvHayLpRqK7BDZt2rRoA7ef1s9152/i5cnCwi8OnH3aQEWj9MvfdDojr0xhFu2r/JYtgwxvXtPQe12zYyNnre+ve/6t2wb58IVbGd48WBo7b/Ma3n/BZl63YTXvPOc0AN521jpuvGgbbzzjVQ3/Ho7jOLVQHMzqvkD6I/DqGqduIQrw/wMM+Aqwwcw+KOm7wENm9uPwHncBvwP+DXzVzC4O4xcCnzezyxcydHh42Hbt2tXwL+Y4juOApN1mNlw9vmDmHwfqBn7AncBvw5cHgTMTpzcCh8L4xhrjjuM4zkmk1WqfDYkvrwb2heMHgJ2ScpK2Ek3sPmpmI8CEpAtClc/1wP2t2OA4juM0T6ua/9clnUsk+xwAPgJgZk9K+gXwd2AG+LiZxaU2HwPuAVYRTfSe0Mlex3EcZy4Lav7tgmv+juM4zVNP8/fVQo7jOB2IB3/HcZwOxIO/4zhOB+LB33EcpwNZNhO+kg4D/1rkt68jWozWziwHG8HtXEqWg42wPOxcDjZCOnZuNrOh6sFlE/xbQdKuWrPd7cRysBHczqVkOdgIy8PO5WAjtJedLvs4juN0IB78HcdxOpBOCf53pG1AAywHG8HtXEqWg42wPOxcDjZCG9nZEZq/4ziOU0mnZP6O4zhOAg/+juM4HciKDv6SLgkN5PdLuilte2IknSnpz5KekvSkpE+G8Vsl/VfSnvC4rA1sPSDpiWDPrjA2KOkPkp4Jz421NDsx9r024a89kl6R9Kl28KWkuyW9JGlfYqyu7yTdHD6rT0t6d4o2fkPSPyT9TdKvJZ0axrdImkz49PaTYeM8dta9xmn4ch47f56w8YCkPWE8NX8CYGYr8gFkgGeBbUAPsBc4J227gm0bgB3heAD4J3AOcCvw2bTtq7L1ALCuauzrwE3h+Cbga2nbmbjmLwCb28GXwEXADmDfQr4L138vkAO2hs9uJiUb3wVkw/HXEjZuSb6uDXxZ8xqn5ct6dlad/ybwpbT9aWYrOvM/H9hvZs+ZWQG4l6ixfOqY2YiZPR6OJ4CnmKeXcRtyJfCDcPwD4KoUbUnyDuBZM1vsSvAlxcz+AoxVDdfz3ZXAvWY2bWbPA/uJPsMn3UYz+72ZzYQvH6ay+14q1PFlPVLxJcxvZ2hg9V7gZyfDloVYycH/DOA/ia/nbRafFpK2AG8GHglDnwi323enKackMOD3knZLujGMnWZRVzbC8/rUrKtkJ5V/WO3mS6jvu3b9vH6QyoZLWyX9VdKDoQd32tS6xu3qywuBF83smcRYav5cycFfNcbaqq5VUj9wH/ApM3sFuA14DXAuMEJ0i5g2bzOzHcClwMclXZS2QbWQ1ANcAfwyDLWjL+ej7T6vkm4h6sT3kzA0AmwyszcDnwZ+Kml1WvZR/xq3nS8D11GZnKTqz5Uc/Os1kW8LJHUTBf6fmNmvAMzsRTMrmtkscCcn6VZ1PszsUHh+Cfg1kU0vKvRvDs8vpWdhiUuBx83sRWhPXwbq+a6tPq+SbgDeA7zPgkAdZJTRcLybSEs/Oy0b57nGbeVLAElZ4Brg5/FY2v5cycH/MWC7pK0hK9xJ1Fg+dYL2dxfwlJl9KzG+IfGyq4F91d97MpHUJ2kgPiaaCNxH5McbwstuAO5Px8IKKrKqdvNlgnq+ewDYKSknaSuwHXg0BfuQdAnwBeAKMzuWGB+SlAnH24KNz6VhY7Ch3jVuG18muBj4h5kdjAdS92daM80n4wFcRlRJ8yxwS9r2JOx6O9Ft6N+APeFxGfAj4Ikw/gCwIWU7txFVTewFnox9CKwF/gQ8E54HU7azFxgFXpUYS92XRP+MRoDjRNnoh+bzHXBL+Kw+DVyaoo37iTTz+LN5e3jtteFzsBd4HLg8ZV/WvcZp+LKenWH8HuCjVa9NzZ9m5ts7OI7jdCIrWfZxHMdx6uDB33EcpwPx4O84jtOBePB3HMfpQDz4O47jdCAe/B3HcToQD/6O4zgdyP8BO1Jv7al7zzIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
